optional find by nom. annotation (close #143)

Author: dimus

cmd/flags.go

@@ -95,6 +95,13 @@ func allMatchesFlag(cmd *cobra.Command) {
 	}
 }
 
+func findByAnnotFlag(cmd *cobra.Command) {
+	b, _ := cmd.Flags().GetBool("find-by-annotation")
+	if b {
+		opts = append(opts, config.OptWithFindByAnnotation(b))
+	}
+}
+
 func oddsDetailsFlag(cmd *cobra.Command) {
 	b, _ := cmd.Flags().GetBool("details-odds")
 	if b {

cmd/gnfinder.yml

@@ -71,6 +71,12 @@
 #
 # WithBayesOddsDetails: false
 
+# WithFindByAnnotation allows to detect names by existence of a
+# nomenclatural annotation. If it is true, dictionaries do not prevent
+# detection of a name.
+#
+# WithFindByAnnotation: false
+
 # WithOddsAdjustment can be set to true to adjust calculated odds using the
 # ratio of scientific names found in text to the number of capitalized
 # words.

cmd/root.go

@@ -65,6 +65,7 @@ type cfgData struct {
 	WithAllMatches       bool
 	WithAmbiguousNames   bool
 	WithBayesOddsDetails bool
+	WithFindByAnnotation bool
 	WithOddsAdjustment   bool
 	WithPlainInput       bool
 	WithPositionInBytes  bool
@@ -120,6 +121,7 @@ verification results.
 		inputOnlyFlag(cmd)
 		langFlag(cmd)
 		allMatchesFlag(cmd)
+		findByAnnotFlag(cmd)
 		oddsDetailsFlag(cmd)
 		plainInputFlag(cmd)
 		sourcesFlag(cmd)
@@ -194,6 +196,9 @@ func init() {
 		"show details of odds calculation.")
 	rootCmd.Flags().StringP("verifier-url", "e", "",
 		"custom URL for name-verification service.")
+	rootCmd.Flags().BoolP("find-by-annotation", "F", false,
+		`if there is a nomenclatural annotation ('sp. nov.' etc),
+a name will be detected.`)
 	rootCmd.Flags().StringP("format", "f", "",
 		`Format of the output: "compact", "pretty", "csv".
   compact: compact JSON,
@@ -272,6 +277,7 @@ func initConfig() {
 	_ = viper.BindEnv("WithAmbiguousNames", "GNF_WITH_AMBIGUOUS_NAMES")
 	_ = viper.BindEnv("WithAllMatches", "GNF_WITH_ALL_MATCHES")
 	_ = viper.BindEnv("WithBayesOddsDetails", "GNF_WITH_BAYES_ODDS_DETAILS")
+	_ = viper.BindEnv("WithFindByAnnotation", "GNF_WITH_FIND_BY_ANNOTATION")
 	_ = viper.BindEnv("WithOddsAdjustment", "GNF_WITH_ODDS_ADJUSTMENT")
 	_ = viper.BindEnv("WithPlainInput", "GNF_WITH_PLAIN_INPUT")
 	_ = viper.BindEnv("WithPositionInBytes", "GNF_WITH_POSITION_IN_BYTES")
@@ -362,6 +368,14 @@ func getOpts() {
 		opts = append(opts, config.OptWithBayesOddsDetails(true))
 	}
 
+	if cfgCli.WithFindByAnnotation {
+		opts = append(opts, config.OptWithFindByAnnotation(true))
+	}
+
+	if cfgCli.WithOddsAdjustment {
+		opts = append(opts, config.OptWithOddsAdjustment(true))
+	}
+
 	if cfgCli.WithPlainInput {
 		opts = append(opts, config.OptWithPlainInput(true))
 	}
@@ -370,10 +384,6 @@ func getOpts() {
 		opts = append(opts, config.OptWithPositonInBytes(true))
 	}
 
-	if cfgCli.WithOddsAdjustment {
-		opts = append(opts, config.OptWithOddsAdjustment(true))
-	}
-
 	if cfgCli.WithUniqueNames {
 		opts = append(opts, config.OptWithUniqueNames(true))
 	}

pkg/config/config.go

@@ -75,6 +75,11 @@ type Config struct {
 	// WithBayesOddsDetails show in detail how odds are calculated.
 	WithBayesOddsDetails bool
 
+	// WithFindByAnnotation allows to detect names by existence of a
+	// nomenclatural annotation. If it is true, dictionaries do not prevent
+	// detection of a name.
+	WithFindByAnnotation bool
+
 	// WithOddsAdjustment can be set to true to adjust calculated odds using the
 	// ratio of scientific names found in text to the number of capitalized
 	// words.
@@ -208,6 +213,14 @@ func OptWithBayesOddsDetails(b bool) Option {
 	}
 }
 
+// OptWithFindByAnnotation option to allow detect names solely by their
+// nomenclatural annotation.
+func OptWithFindByAnnotation(b bool) Option {
+	return func(cfg *Config) {
+		cfg.WithFindByAnnotation = b
+	}
+}
+
 // OptWithOddsAdjustment is an option that triggers recalculation of prior odds
 // using number of found names divided by number of all name candidates.
 func OptWithOddsAdjustment(b bool) Option {

pkg/config/config_test.go

@@ -16,17 +16,17 @@ func TestConfig(t *testing.T) {
 
 	t.Run("returns new Config object", func(t *testing.T) {
 		cfg := config.New()
-		assert.Equal(t, cfg.Language, lang.English)
-		assert.Equal(t, cfg.LanguageDetected, "")
-		assert.Equal(t, cfg.TokensAround, 0)
+		assert.Equal(t, lang.English, cfg.Language)
+		assert.Equal(t, "", cfg.LanguageDetected)
+		assert.Equal(t, 0, cfg.TokensAround)
 		assert.True(t, cfg.WithBayes)
 		assert.False(t, cfg.WithPositionInBytes)
 	})
 
 	t.Run("takes language", func(t *testing.T) {
 		cfg := config.New(config.OptLanguage(lang.English))
-		assert.Equal(t, cfg.Language, lang.English)
-		assert.Equal(t, cfg.LanguageDetected, "")
+		assert.Equal(t, lang.English, cfg.Language)
+		assert.Equal(t, "", cfg.LanguageDetected)
 	})
 
 	t.Run("sets bayes", func(t *testing.T) {
@@ -41,19 +41,24 @@ func TestConfig(t *testing.T) {
 
 	t.Run("sets tokens number", func(t *testing.T) {
 		cfg := config.New(config.OptTokensAround(4))
-		assert.Equal(t, cfg.TokensAround, 4)
+		assert.Equal(t, 4, cfg.TokensAround)
+	})
+
+	t.Run("sets find by annotation", func(t *testing.T) {
+		cfg := config.New(config.OptWithFindByAnnotation(true))
+		assert.Equal(t, true, cfg.WithFindByAnnotation)
 	})
 
 	t.Run("does not set 'bad' tokens number", func(t *testing.T) {
 		cfg := config.New(config.OptTokensAround(-1))
-		assert.Equal(t, cfg.TokensAround, 0)
+		assert.Equal(t, 0, cfg.TokensAround)
 		cfg = config.New(config.OptTokensAround(10))
-		assert.Equal(t, cfg.TokensAround, 5)
+		assert.Equal(t, 5, cfg.TokensAround)
 	})
 
 	t.Run("sets bayes' threshold", func(t *testing.T) {
 		cfg := config.New(config.OptBayesOddsThreshold(200))
-		assert.Equal(t, cfg.BayesOddsThreshold, 200.0)
+		assert.Equal(t, 200.0, cfg.BayesOddsThreshold)
 	})
 
 	t.Run("sets several options", func(t *testing.T) {
@@ -62,7 +67,7 @@ func TestConfig(t *testing.T) {
 			config.OptLanguage(lang.German),
 		}
 		cfg := config.New(opts...)
-		assert.Equal(t, cfg.Language, lang.German)
+		assert.Equal(t, lang.German, cfg.Language)
 		assert.True(t, cfg.WithBayes)
 	})
 
@@ -81,11 +86,11 @@ func TestConfig(t *testing.T) {
 
 		for _, v := range tests {
 			l, err := lang.New(v.lang)
-			assert.Equal(t, err != nil, v.hasErr, v.msg)
+			assert.Equal(t, v.hasErr, err != nil, v.msg)
 			langOpt := config.OptLanguage(l)
 			opts := []config.Option{langOpt}
 			cfg := config.New(opts...)
-			assert.Equal(t, cfg.Language, v.langCfg, v.msg)
+			assert.Equal(t, v.langCfg, cfg.Language, v.msg)
 		}
 	})
 }

pkg/ent/heuristic/heuristic.go

@@ -11,7 +11,7 @@ import (
 // tokens and sets up token's indices. Indices determine if a token is a
 // potential unimonial, binomial or trinomial. Then if fills out signfificant
 // number of features pertained to the token.
-func TagTokens(ts []token.TokenSN, d *dict.Dictionary) {
+func TagTokens(ts []token.TokenSN, d *dict.Dictionary, withAnnot bool) {
 	l := len(ts)
 
 	for i := range ts {

pkg/ent/heuristic/heuristic_test.go

@@ -17,7 +17,7 @@ func TestHeuristic(t *testing.T) {
          randomly... Pardosa is a very nice when it is not sad. Drosophila
          (Sophophora) melanogaster disagrees!`)
 	ts := token.Tokenize(txt)
-	heuristic.TagTokens(ts, dictionary)
+	heuristic.TagTokens(ts, dictionary, false)
 	tests := map[int]struct {
 		name     string
 		decision token.Decision

pkg/ent/nlp/bayes_test.go

@@ -26,7 +26,7 @@ cheilum, 1 5s. per doz.
 Conostylis americana, 2i. 6d.
 			`)
 	tokens := token.Tokenize(txt)
-	heuristic.TagTokens(tokens, dictionary)
+	heuristic.TagTokens(tokens, dictionary, false)
 	nb := weights[lang.English]
 
 	tkn := tokens[10]

pkg/gnfinder.go

@@ -64,7 +64,7 @@ func (gnf gnfinder) Find(file, txt string) output.Output {
 		gnf.Language, gnf.LanguageDetected = lang.DetectLanguage(text)
 	}
 
-	heuristic.TagTokens(tokens, gnf.Dictionary)
+	heuristic.TagTokens(tokens, gnf.Dictionary, gnf.WithFindByAnnotation)
 	if gnf.WithBayes {
 		nb := gnf.bayesWeights[gnf.Language]
 		nlp.TagTokens(tokens, gnf.Dictionary, nb, gnf.BayesOddsThreshold)

tools/training/trainer.go

@@ -113,7 +113,7 @@ func processText(t *TextData, d *dict.Dictionary) []feature.ClassFeatures {
 	var lfs, lfsText []feature.ClassFeatures
 	var nd NameData
 	ts := token.Tokenize(t.Text)
-	heuristic.TagTokens(ts, d)
+	heuristic.TagTokens(ts, d, false)
 	l := len(t.NamesPositions)
 	var nameIdx, i int
 	for {