optional name detection by annotation (close #143)

Author: dimus

pkg/ent/token/features.go

@@ -25,6 +25,11 @@ type Features struct {
 	// Abbr feature: token ends with a period.
 	Abbr bool
 
+	// HasAnnotation is true if there is an apparent nomenclatural annotation
+	// attached to a name candidate. This feature can only be set for the
+	// first token of a candidate.
+	HasAnnotation bool
+
 	// PotentialBinomialGenus feature: the token might be a genus of name.
 	PotentialBinomialGenus bool
 

pkg/ent/token/token.go

@@ -13,8 +13,8 @@ import (
 	"github.com/gnames/gnfinder/pkg/io/dict"
 )
 
-// tokenSN represents a word separated by spaces in a text. Words that are
-// split by new lines are concatenated.
+// tokenSN (a token for a 'Scientific Name') represents a word separated by
+// spaces in a text. Words that are split by new lines are concatenated.
 type tokenSN struct {
 	gner.TokenNER
 
@@ -218,10 +218,13 @@ func checkRank(t TokenSN, d *dict.Dictionary) bool {
 
 // UpperIndex takes an index of a token and length of the tokens slice and
 // returns an upper index of what could be a slice of a name. We expect that
-// that most of the names will fit into 5 words. Other cases would require
-// more thorough algorithims that we can run later as plugins.
+// that most of the names will fit into 5 words. We also expect that annotation
+// will be located not further away than 10 words away.
+// Other cases would require more thorough algorithims that we can run
+// later as plugins.
 func UpperIndex(i int, l int) int {
-	upperIndex := i + 5
+	// 10 here is 5 tokens for name, 5 tokens for annotation
+	upperIndex := i + 10
 	if l < upperIndex {
 		upperIndex = l
 	}