Merge pull request #243 from vmarkovtsev/master

Add TyposDatasetBuilder

Author: vmarkovtsev

core.go

@@ -9,7 +9,8 @@ import (
 	"gopkg.in/src-d/hercules.v9/internal/plumbing/identity"
 	"gopkg.in/src-d/hercules.v9/internal/plumbing/uast"
 	"gopkg.in/src-d/hercules.v9/internal/yaml"
-	"gopkg.in/src-d/hercules.v9/leaves"
+	_ "gopkg.in/src-d/hercules.v9/leaves"          // add burndown and other analyses
+	_ "gopkg.in/src-d/hercules.v9/leaves/research" // add "research" analyses
 )
 
 // ConfigurationOptionType represents the possible types of a ConfigurationOption's value.
@@ -171,8 +172,3 @@ func PathifyFlagValue(flag *pflag.Flag) {
 func EnablePathFlagTypeMasquerade() {
 	core.EnablePathFlagTypeMasquerade()
 }
-
-func init() {
-	// hack to link with .leaves
-	_ = leaves.BurndownAnalysis{}
-}

internal/pb/pb.pb.go

@@ -165,6 +165,18 @@ message CommitsAnalysisResults {
     repeated string author_index = 2;
 }
 
+message Typo {
+    string wrong = 1;
+    string correct = 2;
+    string commit = 3;
+    string file = 4;
+    int32 line = 5;
+}
+
+message TyposDataset {
+    repeated Typo typos = 1;
+}
+
 message AnalysisResults {
     Metadata header = 1;
     // the mapped values are dynamic messages which require the second parsing pass.

internal/pb/pb.proto

@@ -32,11 +32,12 @@ import (
 // It is a PipelineItem.
 type Extractor struct {
 	core.NoopMerger
-	Endpoint       string
-	Context        func() (context.Context, context.CancelFunc)
-	PoolSize       int
-	FailOnErrors   bool
-	ProcessedFiles map[string]int
+	Endpoint              string
+	Context               func() (context.Context, context.CancelFunc)
+	PoolSize              int
+	FailOnErrors          bool
+	ProcessedFiles        map[string]int
+	IgnoredMissingDrivers map[string]bool
 
 	clients []*bblfsh.Client
 	pool    *tunny.Pool
@@ -45,22 +46,36 @@ type Extractor struct {
 const (
 	// ConfigUASTEndpoint is the name of the configuration option (Extractor.Configure())
 	// which sets the Babelfish server address.
-	ConfigUASTEndpoint = "ConfigUASTEndpoint"
+	ConfigUASTEndpoint = "UAST.Endpoint"
 	// ConfigUASTTimeout is the name of the configuration option (Extractor.Configure())
 	// which sets the maximum amount of time to wait for a Babelfish server response.
-	ConfigUASTTimeout = "ConfigUASTTimeout"
+	ConfigUASTTimeout = "UAST.Timeout"
 	// ConfigUASTPoolSize is the name of the configuration option (Extractor.Configure())
 	// which sets the number of goroutines to run for UAST parse queries.
-	ConfigUASTPoolSize = "ConfigUASTPoolSize"
+	ConfigUASTPoolSize = "UAST.PoolSize"
 	// ConfigUASTFailOnErrors is the name of the configuration option (Extractor.Configure())
 	// which enables early exit in case of any Babelfish UAST parsing errors.
-	ConfigUASTFailOnErrors = "ConfigUASTFailOnErrors"
+	ConfigUASTFailOnErrors = "UAST.FailOnErrors"
+	// ConfigUASTIgnoreMissingDrivers is the name of the configuration option (Extractor.Configure())
+	// which sets the ignored missing driver names.
+	ConfigUASTIgnoreMissingDrivers = "UAST.IgnoreMissingDrivers"
+	// DefaultBabelfishEndpoint is the default address of the Babelfish parsing server.
+	DefaultBabelfishEndpoint = "0.0.0.0:9432"
+	// DefaultBabelfishTimeout is the default value of the RPC timeout in seconds.
+	DefaultBabelfishTimeout = 20
 	// FeatureUast is the name of the Pipeline feature which activates all the items related to UAST.
 	FeatureUast = "uast"
 	// DependencyUasts is the name of the dependency provided by Extractor.
 	DependencyUasts = "uasts"
 )
 
+var (
+	// DefaultBabelfishWorkers is the default number of parsing RPC goroutines.
+	DefaultBabelfishWorkers = runtime.NumCPU() * 2
+	// DefaultIgnoredMissingDrivers is the languages which are ignored if the Babelfish driver is missing.
+	DefaultIgnoredMissingDrivers = []string{"markdown", "text", "yaml", "json"}
+)
+
 type uastTask struct {
 	Lock   *sync.RWMutex
 	Dest   map[plumbing.Hash]nodes.Node
@@ -117,22 +132,27 @@ func (exr *Extractor) ListConfigurationOptions() []core.ConfigurationOption {
 		Description: "How many days there are in a single band.",
 		Flag:        "bblfsh",
 		Type:        core.StringConfigurationOption,
-		Default:     "0.0.0.0:9432"}, {
+		Default:     DefaultBabelfishEndpoint}, {
 		Name:        ConfigUASTTimeout,
 		Description: "Babelfish's server timeout in seconds.",
 		Flag:        "bblfsh-timeout",
 		Type:        core.IntConfigurationOption,
-		Default:     20}, {
+		Default:     DefaultBabelfishTimeout}, {
 		Name:        ConfigUASTPoolSize,
 		Description: "Number of goroutines to extract UASTs.",
 		Flag:        "bblfsh-pool-size",
 		Type:        core.IntConfigurationOption,
-		Default:     runtime.NumCPU() * 2}, {
+		Default:     DefaultBabelfishWorkers}, {
 		Name:        ConfigUASTFailOnErrors,
 		Description: "Panic if there is a UAST extraction error.",
 		Flag:        "bblfsh-fail-on-error",
 		Type:        core.BoolConfigurationOption,
-		Default:     false},
+		Default:     false}, {
+		Name:        ConfigUASTIgnoreMissingDrivers,
+		Description: "Do not warn about missing drivers for the specified languages.",
+		Flag:        "bblfsh-ignored-drivers",
+		Type:        core.StringsConfigurationOption,
+		Default:     DefaultIgnoredMissingDrivers},
 	}
 	return options[:]
 }
@@ -154,6 +174,12 @@ func (exr *Extractor) Configure(facts map[string]interface{}) error {
 	if val, exists := facts[ConfigUASTFailOnErrors].(bool); exists {
 		exr.FailOnErrors = val
 	}
+	if val, exists := facts[ConfigUASTIgnoreMissingDrivers].([]string); exists {
+		exr.IgnoredMissingDrivers = map[string]bool{}
+		for _, name := range val {
+			exr.IgnoredMissingDrivers[name] = true
+		}
+	}
 	return nil
 }
 
@@ -162,9 +188,16 @@ func (exr *Extractor) Configure(facts map[string]interface{}) error {
 func (exr *Extractor) Initialize(repository *git.Repository) error {
 	if exr.Context == nil {
 		exr.Context = func() (context.Context, context.CancelFunc) {
-			return context.Background(), nil
+			return context.WithTimeout(context.Background(),
+				time.Duration(DefaultBabelfishTimeout)*time.Second)
 		}
 	}
+	if exr.Endpoint == "" {
+		exr.Endpoint = DefaultBabelfishEndpoint
+	}
+	if exr.PoolSize == 0 {
+		exr.PoolSize = DefaultBabelfishWorkers
+	}
 	poolSize := exr.PoolSize
 	if poolSize == 0 {
 		poolSize = runtime.NumCPU()
@@ -196,6 +229,12 @@ func (exr *Extractor) Initialize(repository *git.Repository) error {
 		panic("UAST goroutine pool was not created")
 	}
 	exr.ProcessedFiles = map[string]int{}
+	if exr.IgnoredMissingDrivers == nil {
+		exr.IgnoredMissingDrivers = map[string]bool{}
+		for _, name := range DefaultIgnoredMissingDrivers {
+			exr.IgnoredMissingDrivers[name] = true
+		}
+	}
 	return nil
 }
 
@@ -250,7 +289,7 @@ func (exr *Extractor) Consume(deps map[string]interface{}) (map[string]interface
 		if exr.FailOnErrors {
 			return nil, errors.New(joined)
 		}
-		fmt.Fprintln(os.Stderr, joined)
+		log.Println(joined)
 	}
 	return map[string]interface{}{DependencyUasts: uasts}, nil
 }
@@ -284,6 +323,11 @@ func (exr *Extractor) extractTask(client *bblfsh.Client, data interface{}) inter
 	task.Lock.Lock()
 	defer task.Lock.Unlock()
 	if err != nil {
+		for lang := range exr.IgnoredMissingDrivers {
+			if strings.HasSuffix(err.Error(), "\""+lang+"\"") {
+				return nil
+			}
+		}
 		*task.Errors = append(*task.Errors,
 			fmt.Errorf("\nfile %s, blob %s: %v", task.Name, task.Hash.String(), err))
 		return nil

internal/pb/pb_pb2.py

@@ -50,11 +50,12 @@ func TestUASTExtractorMeta(t *testing.T) {
 	assert.Equal(t, exr.Requires()[0], items.DependencyTreeChanges)
 	assert.Equal(t, exr.Requires()[1], items.DependencyBlobCache)
 	opts := exr.ListConfigurationOptions()
-	assert.Len(t, opts, 4)
+	assert.Len(t, opts, 5)
 	assert.Equal(t, opts[0].Name, ConfigUASTEndpoint)
 	assert.Equal(t, opts[1].Name, ConfigUASTTimeout)
 	assert.Equal(t, opts[2].Name, ConfigUASTPoolSize)
 	assert.Equal(t, opts[3].Name, ConfigUASTFailOnErrors)
+	assert.Equal(t, opts[4].Name, ConfigUASTIgnoreMissingDrivers)
 	feats := exr.Features()
 	assert.Len(t, feats, 1)
 	assert.Equal(t, feats[0], FeatureUast)
@@ -68,11 +69,13 @@ func TestUASTExtractorConfiguration(t *testing.T) {
 	facts[ConfigUASTTimeout] = 15
 	facts[ConfigUASTPoolSize] = 7
 	facts[ConfigUASTFailOnErrors] = true
+	facts[ConfigUASTIgnoreMissingDrivers] = []string{"test"}
 	exr.Configure(facts)
 	assert.Equal(t, exr.Endpoint, facts[ConfigUASTEndpoint])
 	assert.NotNil(t, exr.Context)
 	assert.Equal(t, exr.PoolSize, facts[ConfigUASTPoolSize])
 	assert.Equal(t, exr.FailOnErrors, true)
+	assert.Equal(t, exr.IgnoredMissingDrivers, map[string]bool{"test": true})
 }
 
 func TestUASTExtractorRegistration(t *testing.T) {
@@ -92,7 +95,7 @@ func TestUASTExtractorNoBabelfish(t *testing.T) {
 
 func TestUASTExtractorConsume(t *testing.T) {
 	exr := fixtureUASTExtractor()
-	changes := make(object.Changes, 3)
+	changes := make(object.Changes, 4)
 	// 2b1ed978194a94edeabbca6de7ff3b5771d4d665
 	treeFrom, _ := test.Repository.TreeObject(plumbing.NewHash(
 		"96c6ece9b2f3c7c51b83516400d278dea5605100"))
@@ -136,13 +139,24 @@ func TestUASTExtractorConsume(t *testing.T) {
 		},
 	},
 	}
+	changes[3] = &object.Change{From: object.ChangeEntry{}, To: object.ChangeEntry{
+		Name: "README.md",
+		Tree: treeTo,
+		TreeEntry: object.TreeEntry{
+			Name: "README.md",
+			Mode: 0100644,
+			Hash: plumbing.NewHash("5248c86995f6d60eb57730da18b5e020a4341863"),
+		},
+	},
+	}
 	cache := map[plumbing.Hash]*items.CachedBlob{}
 	for _, hash := range []string{
 		"baa64828831d174f40140e4b3cfa77d1e917a2c1",
 		"5d78f57d732aed825764347ec6f3ab74d50d0619",
 		"c29112dbd697ad9b401333b80c18a63951bc18d9",
 		"f7d918ec500e2f925ecde79b51cc007bac27de72",
 		"81f2b6d1fa5357f90e9dead150cd515720897545",
+		"5248c86995f6d60eb57730da18b5e020a4341863",
 	} {
 		AddHash(t, cache, hash)
 	}
@@ -158,6 +172,12 @@ func TestUASTExtractorConsume(t *testing.T) {
 	assert.Len(t, res[DependencyUasts], 1)
 	assert.Nil(t, err)
 
+	exr.FailOnErrors = true
+	res, err = exr.Consume(deps)
+	assert.Nil(t, res)
+	assert.NotNil(t, err)
+	exr.FailOnErrors = false
+
 	hash := plumbing.NewHash("5d78f57d732aed825764347ec6f3ab74d50d0619")
 	changes[1] = &object.Change{From: object.ChangeEntry{}, To: object.ChangeEntry{
 		Name: "labours.py",
@@ -176,6 +196,17 @@ func TestUASTExtractorConsume(t *testing.T) {
 	uasts := res[DependencyUasts].(map[plumbing.Hash]nodes.Node)
 	assert.Equal(t, len(uasts), 1)
 	assert.Equal(t, len(uasts[hash].(nodes.Object)["body"].(nodes.Array)), 24)
+
+	exr.IgnoredMissingDrivers = map[string]bool{}
+	changes[2] = changes[3]
+	deps[items.DependencyTreeChanges] = changes[:3]
+	res, err = exr.Consume(deps)
+	assert.Nil(t, err)
+	exr.FailOnErrors = true
+	res, err = exr.Consume(deps)
+	assert.Nil(t, res)
+	assert.NotNil(t, err)
+	exr.FailOnErrors = false
 }
 
 func TestUASTExtractorFork(t *testing.T) {