Skip to content

Commit

Permalink
Merge branch 'master' into 2512-local
Browse files Browse the repository at this point in the history
  • Loading branch information
arfon committed Mar 17, 2016
2 parents ed4bbe9 + e6dabd5 commit d377e23
Show file tree
Hide file tree
Showing 334 changed files with 53,794 additions and 1,059 deletions.
89 changes: 73 additions & 16 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,6 @@
[submodule "vendor/grammars/Sublime-REBOL"]
path = vendor/grammars/Sublime-REBOL
url = https://github.com/Oldes/Sublime-REBOL
[submodule "vendor/grammars/Sublime-Inform"]
path = vendor/grammars/Sublime-Inform
url = https://github.com/PogiNate/Sublime-Inform
[submodule "vendor/grammars/autoitv3-tmbundle"]
path = vendor/grammars/autoitv3-tmbundle
url = https://github.com/Red-Nova-Technologies/autoitv3-tmbundle
Expand Down Expand Up @@ -85,6 +82,9 @@
[submodule "vendor/grammars/language-shellscript"]
path = vendor/grammars/language-shellscript
url = https://github.com/atom/language-shellscript
[submodule "vendor/grammars/language-supercollider"]
path = vendor/grammars/language-supercollider
url = https://github.com/supercollider/language-supercollider
[submodule "vendor/grammars/language-yaml"]
path = vendor/grammars/language-yaml
url = https://github.com/atom/language-yaml
Expand Down Expand Up @@ -169,9 +169,6 @@
[submodule "vendor/grammars/sublime-idris"]
path = vendor/grammars/sublime-idris
url = https://github.com/laughedelic/sublime-idris
[submodule "vendor/grammars/sublime-better-typescript"]
path = vendor/grammars/sublime-better-typescript
url = https://github.com/lavrton/sublime-better-typescript
[submodule "vendor/grammars/moonscript-tmbundle"]
path = vendor/grammars/moonscript-tmbundle
url = https://github.com/leafo/moonscript-tmbundle
Expand Down Expand Up @@ -256,9 +253,6 @@
[submodule "vendor/grammars/SublimeXtend"]
path = vendor/grammars/SublimeXtend
url = https://github.com/staltz/SublimeXtend
[submodule "vendor/grammars/Stata.tmbundle"]
path = vendor/grammars/Stata.tmbundle
url = https://github.com/statatmbundle/Stata.tmbundle
[submodule "vendor/grammars/Vala-TMBundle"]
path = vendor/grammars/Vala-TMBundle
url = https://github.com/technosophos/Vala-TMBundle
Expand Down Expand Up @@ -346,9 +340,6 @@
[submodule "vendor/grammars/latex.tmbundle"]
path = vendor/grammars/latex.tmbundle
url = https://github.com/textmate/latex.tmbundle
[submodule "vendor/grammars/less.tmbundle"]
path = vendor/grammars/less.tmbundle
url = https://github.com/textmate/less.tmbundle
[submodule "vendor/grammars/lilypond.tmbundle"]
path = vendor/grammars/lilypond.tmbundle
url = https://github.com/textmate/lilypond.tmbundle
Expand Down Expand Up @@ -397,9 +388,6 @@
[submodule "vendor/grammars/processing.tmbundle"]
path = vendor/grammars/processing.tmbundle
url = https://github.com/textmate/processing.tmbundle
[submodule "vendor/grammars/prolog.tmbundle"]
path = vendor/grammars/prolog.tmbundle
url = https://github.com/textmate/prolog.tmbundle
[submodule "vendor/grammars/python-django.tmbundle"]
path = vendor/grammars/python-django.tmbundle
url = https://github.com/textmate/python-django.tmbundle
Expand Down Expand Up @@ -655,7 +643,7 @@
url = https://github.com/SRI-CSL/SMT.tmbundle.git
[submodule "vendor/grammars/language-crystal"]
path = vendor/grammars/language-crystal
url = https://github.com/k2b6s9j/language-crystal
url = https://github.com/atom-crystal/language-crystal
[submodule "vendor/grammars/language-xbase"]
path = vendor/grammars/language-xbase
url = https://github.com/hernad/atom-language-harbour
Expand All @@ -665,3 +653,72 @@
[submodule "vendor/grammars/pawn-sublime-language"]
path = vendor/grammars/pawn-sublime-language
url = https://github.com/Southclaw/pawn-sublime-language.git
[submodule "vendor/grammars/atom-language-purescript"]
path = vendor/grammars/atom-language-purescript
url = https://github.com/purescript-contrib/atom-language-purescript
[submodule "vendor/grammars/vue-syntax-highlight"]
path = vendor/grammars/vue-syntax-highlight
url = https://github.com/vuejs/vue-syntax-highlight
[submodule "vendor/grammars/st2-zonefile"]
path = vendor/grammars/st2-zonefile
url = https://github.com/sixty4k/st2-zonefile
[submodule "vendor/grammars/sublimeprolog"]
path = vendor/grammars/sublimeprolog
url = https://github.com/alnkpa/sublimeprolog
[submodule "vendor/grammars/sublime-aspectj"]
path = vendor/grammars/sublime-aspectj
url = https://github.com/pchaigno/sublime-aspectj
[submodule "vendor/grammars/sublime-typescript"]
path = vendor/grammars/sublime-typescript
url = https://github.com/Microsoft/TypeScript-Sublime-Plugin
[submodule "vendor/grammars/sublime-pony"]
path = vendor/grammars/sublime-pony
url = https://github.com/CausalityLtd/sublime-pony
[submodule "vendor/grammars/X10"]
path = vendor/grammars/X10
url = https://github.com/x10-lang/x10-highlighting
[submodule "vendor/grammars/language-babel"]
path = vendor/grammars/language-babel
url = https://github.com/gandm/language-babel
[submodule "vendor/grammars/UrWeb-Language-Definition"]
path = vendor/grammars/UrWeb-Language-Definition
url = https://github.com/gwalborn/UrWeb-Language-Definition.git
[submodule "vendor/grammars/Stata.tmbundle"]
path = vendor/grammars/Stata.tmbundle
url = https://github.com/pschumm/Stata.tmbundle
[submodule "vendor/grammars/FreeMarker.tmbundle"]
path = vendor/grammars/FreeMarker.tmbundle
url = https://github.com/freemarker/FreeMarker.tmbundle
[submodule "vendor/grammars/MagicPython"]
path = vendor/grammars/MagicPython
url = https://github.com/MagicStack/MagicPython
[submodule "vendor/grammars/language-click"]
path = vendor/grammars/language-click
url = https://github.com/stenverbois/language-click.git
[submodule "vendor/grammars/language-maxscript"]
path = vendor/grammars/language-maxscript
url = https://github.com/Alhadis/language-maxscript
[submodule "vendor/grammars/language-renpy"]
path = vendor/grammars/language-renpy
url = https://github.com/williamd1k0/language-renpy.git
[submodule "vendor/grammars/language-inform7"]
path = vendor/grammars/language-inform7
url = https://github.com/erkyrath/language-inform7
[submodule "vendor/grammars/atom-language-stan"]
path = vendor/grammars/atom-language-stan
url = https://github.com/jrnold/atom-language-stan
[submodule "vendor/grammars/language-yang"]
path = vendor/grammars/language-yang
url = https://github.com/DzonyKalafut/language-yang.git
[submodule "vendor/grammars/perl6fe"]
path = vendor/grammars/perl6fe
url = https://github.com/MadcapJake/language-perl6fe.git
[submodule "vendor/grammars/language-less"]
path = vendor/grammars/language-less
url = https://github.com/atom/language-less.git
[submodule "vendor/grammars/language-povray"]
path = vendor/grammars/language-povray
url = https://github.com/c-lipka/language-povray
[submodule "vendor/grammars/sublime-terra"]
path = vendor/grammars/sublime-terra
url = https://github.com/pyk/sublime-terra
8 changes: 7 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
language: ruby
sudo: false
addons:
apt:
packages:
- libicu-dev
- libicu48
before_install: script/travis/before_install
rvm:
- 1.9.3
- 2.0.0
- 2.1
- 2.2
Expand Down
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ We try only to add new extensions once they have some usage on GitHub. In most c

To add support for a new extension:

0. Add your extension to the language entry in [`languages.yml`][languages].
0. Add your extension to the language entry in [`languages.yml`][languages], keeping the extensions in alphabetical order.
0. Add at least one sample for your extension to the [samples directory][samples] in the correct subdirectory.
0. Open a pull request, linking to a [GitHub search result](https://github.com/search?utf8=%E2%9C%93&q=extension%3Aboot+NOT+nothack&type=Code&ref=searchresults) showing in-the-wild usage.

Expand Down
1 change: 0 additions & 1 deletion Gemfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
source 'https://rubygems.org'
gemspec :name => "github-linguist"
gemspec :name => "github-linguist-grammars"
gem 'byebug' if RUBY_VERSION >= '2.0'
2 changes: 1 addition & 1 deletion LICENSE
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Copyright (c) 2011-2015 GitHub, Inc.
Copyright (c) 2011-2016 GitHub, Inc.

Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
Expand Down
20 changes: 14 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ See [Troubleshooting](#troubleshooting) and [`CONTRIBUTING.md`](/CONTRIBUTING.md

![language stats bar](https://cloud.githubusercontent.com/assets/173/5562290/48e24654-8ddf-11e4-8fe7-735b0ce3a0d3.png)

The Language stats bar is built by aggregating the languages of each file in that repository. If it is reporting a language that you don't expect:
The Language stats bar displays languages percentages for the files in the repository. The percentages are calculated based on the bytes of code for each language as reported by the [List Languages](https://developer.github.com/v3/repos/#list-languages) API. If the bar is reporting a language that you don't expect:

0. Click on the name of the language in the stats bar to see a list of the files that are identified as that language.
0. If you see files that you didn't write, consider moving the files into one of the [paths for vendored code](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml), or use the [manual overrides](#overrides) feature to ignore them.
0. If the files are being misclassified, search for [open issues][issues] to see if anyone else has already reported the issue. Any information you an add, especially links to public repositories, is helpful.
0. If you see files that you didn't write, consider moving the files into one of the [paths for vendored code](/lib/linguist/vendor.yml), or use the [manual overrides](#overrides) feature to ignore them.
0. If the files are being misclassified, search for [open issues][issues] to see if anyone else has already reported the issue. Any information you can add, especially links to public repositories, is helpful.
0. If there are no reported issues of this misclassification, [open an issue][new-issue] and include a link to the repository or a sample of the code that is being misclassified.

## Overrides
Expand All @@ -33,9 +33,9 @@ $ cat .gitattributes
*.rb linguist-language=Java
```

Checking code you didn't write, such as JavaScript libraries, into your git repo is a common practice, but this often inflates your project's language stats and may even cause your project to be labeled as another language. By default, Linguist treats all of the paths defined in [lib/linguist/vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml) as vendored and therefore doesn't include them in the language statistics for a repository. Vendored files are also hidden by default in diffs on github.com.
Checking code you didn't write, such as JavaScript libraries, into your git repo is a common practice, but this often inflates your project's language stats and may even cause your project to be labeled as another language. By default, Linguist treats all of the paths defined in [lib/linguist/vendor.yml](https://github.com/github/linguist/blob/master/lib/linguist/vendor.yml) as vendored and therefore doesn't include them in the language statistics for a repository.

Use the `linguist-vendored` attribute to vendor or un-vendor paths. Please note, overriding the vendored (or un-vendored) status of a file only affects the language statistics for the repository and not the behavior in diffs on github.com.
Use the `linguist-vendored` attribute to vendor or un-vendor paths.

```
$ cat .gitattributes
Expand All @@ -59,6 +59,9 @@ Alternatively, you can use Vim or Emacs style modelines to set the language for

##### Vim
```
# Some examples of various styles:
vim: syntax=java
vim: set syntax=ruby:
vim: set filetype=prolog:
vim: set ft=cpp:
```
Expand Down Expand Up @@ -111,4 +114,9 @@ lib/linguist.rb

Please check out our [contributing guidelines](CONTRIBUTING.md).

##
## License

The language grammars included in this gem are covered by their repositories'
respective licenses. `grammars.yml` specifies the repository for each grammar.

All other files are covered by the MIT license, see `LICENSE`.
8 changes: 2 additions & 6 deletions Rakefile
Original file line number Diff line number Diff line change
Expand Up @@ -40,18 +40,14 @@ task :samples do
end

task :build_gem => :samples do
rm_rf "grammars"
sh "script/convert-grammars"
languages = YAML.load_file("lib/linguist/languages.yml")
File.write("lib/linguist/languages.json", Yajl.dump(languages))
`gem build github-linguist.gemspec`
File.delete("lib/linguist/languages.json")
end

task :build_grammars_gem do
rm_rf "grammars"
sh "script/convert-grammars"
sh "gem", "build", "github-linguist-grammars.gemspec"
end

namespace :benchmark do
benchmark_path = "benchmark/results"

Expand Down
138 changes: 138 additions & 0 deletions bin/git-linguist
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
#!/usr/bin/env ruby

require 'linguist'
require 'rugged'
require 'optparse'
require 'json'
require 'tmpdir'
require 'zlib'

class GitLinguist
def initialize(path, commit_oid, incremental = true)
@repo_path = path
@commit_oid = commit_oid
@incremental = incremental
end

def linguist
if @commit_oid.nil?
raise "git-linguist must be called with a specific commit OID to perform language computation"
end
repo = Linguist::Repository.new(rugged, @commit_oid)

if @incremental && stats = load_language_stats
old_commit_oid, old_stats = stats

# A cache with NULL oid means that we want to froze
# these language stats in place and stop computing
# them (for performance reasons)
return old_stats if old_commit_oid == NULL_OID
repo.load_existing_stats(old_commit_oid, old_stats)
end

result = yield repo

save_language_stats(@commit_oid, repo.cache)
result
end

def load_language_stats
version, oid, stats = load_cache
if version == LANGUAGE_STATS_CACHE_VERSION && oid && stats
[oid, stats]
end
end

def save_language_stats(oid, stats)
cache = [LANGUAGE_STATS_CACHE_VERSION, oid, stats]
write_cache(cache)
end

def clear_language_stats
File.unlink(cache_file)
rescue Errno::ENOENT
end

def disable_language_stats
save_language_stats(NULL_OID, {})
end

protected
NULL_OID = ("0" * 40).freeze

LANGUAGE_STATS_CACHE = 'language-stats.cache'
LANGUAGE_STATS_CACHE_VERSION = "v3:#{Linguist::VERSION}"

def rugged
@rugged ||= Rugged::Repository.bare(@repo_path)
end

def cache_file
File.join(@repo_path, LANGUAGE_STATS_CACHE)
end

def write_cache(object)
return unless File.directory? @repo_path

begin
tmp_path = Dir::Tmpname.make_tmpname(cache_file, nil)
File.open(tmp_path, "wb") do |f|
marshal = Marshal.dump(object)
f.write(Zlib::Deflate.deflate(marshal))
end

File.rename(tmp_path, cache_file)
rescue => e
(File.unlink(tmp_path) rescue nil)
raise e
end
end

def load_cache
marshal = File.open(cache_file, "rb") { |f| Zlib::Inflate.inflate(f.read) }
Marshal.load(marshal)
rescue SystemCallError, ::Zlib::DataError, ::Zlib::BufError, TypeError
nil
end
end


def git_linguist(args)
incremental = true
commit = nil

parser = OptionParser.new do |opts|
opts.banner = "Usage: git-linguist [OPTIONS] stats|breakdown|dump-cache|clear|disable"

opts.on("-f", "--force", "Force a full rescan") { incremental = false }
opts.on("--commit=COMMIT", "Commit to index") { |v| commit = v}
end

parser.parse!(args)

git_dir = `git rev-parse --git-dir`.strip
raise "git-linguist must be ran in a Git repository (#{Dir.pwd})" unless $?.success?
wrapper = GitLinguist.new(git_dir, commit, incremental)

case args.pop
when "stats"
wrapper.linguist do |linguist|
puts JSON.dump(linguist.languages)
end
when "breakdown"
wrapper.linguist do |linguist|
puts JSON.dump(linguist.breakdown_by_file)
end
when "dump-cache"
puts JSON.dump(wrapper.load_language_stats)
when "clear"
wrapper.clear_language_stats
when "disable"
wrapper.disable_language_stats
else
$stderr.print(parser.help)
exit 1
end
end

git_linguist(ARGV)
14 changes: 0 additions & 14 deletions github-linguist-grammars.gemspec

This file was deleted.

Loading

0 comments on commit d377e23

Please sign in to comment.