show ambiguous genera if makes sense (close #113, close #114)

If ambiguous genera is found, unhide them if there are species of
that genera in the text.

Author: dimus

CHANGELOG.md

@@ -2,48 +2,51 @@
 
 ## Unreleased
 
+## [v0.18.1] - 2022-03-01 Tue
+
+- Add [#114]: add an option to show ambiguous uninomials.
+- Add [#113]: show ambiguous genera, if there are species names with them.
+
 ## [v0.18.0] - 2022-02-28 Mon
 
-Add [#117]: bring verificaton in sync with gnames v0.8.0
-Add [#116]: add --all-matches flag to show all verification results.
-Add: update input and output objects and REST API
+- Add [#117]: bring verificaton in sync with gnames v0.8.0
+- Add [#116]: add --all-matches flag to show all verification results.
+- Add: update input and output objects and REST API
      Introducing some backward incompatibility.
      See `https://apidoc.globalnames.org/gnfinder-beta`
 
 ## [v0.17.0] - 2022-01-06
 
-Add [#111]: update bayes calculations.
-
-Add [#110]: update verification process using most recent code.
+- Add [#111]: update bayes calculations.
+- Add [#110]: update verification process using most recent code.
             Stats for kingdoms distribution and the main clade that
             contains most of the names in the text.
             Verification JSON is not fully backward compatible.
-
-Add [#109]: add classification path to CSV and TSV outputs.
+- Add [#109]: add classification path to CSV and TSV outputs.
 
 ## [v0.16.3] - 2021-10-31
 
-Add: update dictionaries with Algaebase and fixes
+- Add: update dictionaries with Algaebase and fixes
 
 ## [v0.16.2] - 2021-10-28
 
-Fix [#108]: remove confuxing red 'x' from web-UI results.
+- Fix [#108]: remove confuxing red 'x' from web-UI results.
 
 ## [v0.16.1] - 2021-10-17
 
-Add [#106]: Add API documentation.
+- Add [#106]: Add API documentation.
 
 ## [v0.16.0] - 2021-06-23
 
-Add [#94]: Add web-based user interface.
+- Add [#94]: Add web-based user interface.
 
 ## [v0.15.5]
 
-Add [#105]: Support for URL name-finding in REST API.
+- Add [#105]: Support for URL name-finding in REST API.
 
 ## [v0.15.4]
 
-Add [#104]: merge petectLanguage to language. It allows to simplify logic for
+- Add [#104]: merge petectLanguage to language. It allows to simplify logic for
             language settings. It also changes API signature for parameters.
             Now parameter "language" recognizes
 
@@ -56,26 +59,26 @@ Add [#104]: merge petectLanguage to language. It allows to simplify logic for
 
 ## [v0.15.3]
 
-Fix [#103]: remove conflict between language and detectlanguage parameters.
+- Fix [#103]: remove conflict between language and detectlanguage parameters.
 
 ## [v0.15.2]
 
-Add: update Echo web framework to v4.5.0
-Fix [#102]: 'language' parameter for REST API.
+- Add: update Echo web framework to v4.5.0
+- Fix [#102]: 'language' parameter for REST API.
 
 ## [v0.15.1]
 
-Fix [#101]: BOM interferes with offsets when -U flag is used.
+- Fix [#101]: BOM interferes with offsets when -U flag is used.
 
 ## [v0.15.0]
 
-Add [#99]: add TSV format and make ouput format an option for REST API.
-Add:       update modules
-Add:       update Go to 1.17
-Add [#98]: an option to return names positions in bytes from the text start
+- Add [#99]: add TSV format and make ouput format an option for REST API.
+- Add:       update modules
+- Add:       update Go to 1.17
+- Add [#98]: an option to return names positions in bytes from the text start
            instead of UTF-8 characters.
 
-Fix [#100]: fix csv/tsv fields number for verification
+- Fix [#100]: fix csv/tsv fields number for verification
 
 ## [v0.14.2]
 

README.md

@@ -164,6 +164,7 @@ settings in both the configuration file and from the flags.
 | TokensAround          | GNF_TOKENS_AROUND           |
 | VerifierURL           | GNF_VERIFIER_URL            |
 | WithAllMatches        | GNF_WITH_ALL_MATCHES        |
+| WithAmbiguousNames    | GNF_WITH_AMBIGUOUS_NAMES    |
 | WithBayesOddsDetails  | GNF_WITH_BAYES_ODDS_DETAILS |
 | WithOddsAdjustment    | GNF_WITH_ODDS_ADJUSTMENT    |
 | WithPlainInput        | GNF_WITH_PLAIN_INPUT        |
@@ -253,6 +254,13 @@ echo "Pomatomus saltator and Parus major" | gnfinder -v -l eng -s "4,12"
 echo "Pomatomus saltator and Parus major" | gnfinder --verify --lang eng --sources "4,12"
 ```
 
+Preserve uninomial names that are also common words.
+
+```bash
+echo "Cancer is a genus" | gnfinder -A
+echo "America is also a genus" | gnfinder --ambiguous-uninomials
+```
+
 Show all matches, not only the best result.
 
 ```bash

config/config.go

@@ -63,6 +63,9 @@ type Config struct {
 	// WithAllMatches sets verification to return all found matches.
 	WithAllMatches bool
 
+	// WithAmbiguousNames shows ambigous uninomial names when true.
+	WithAmbiguousNames bool
+
 	// WithBayes determines if both heuristic and Naive Bayes algorithms run
 	// during the name-finnding.
 	// false - only heuristic algorithms run
@@ -179,6 +182,14 @@ func OptWithAllMatches(b bool) Option {
 	}
 }
 
+// OptWithAmbiguousNames sets WithAmbiguousNames option to show ambiguous
+// uninomials and genera.
+func OptWithAmbiguousNames(b bool) Option {
+	return func(cfg *Config) {
+		cfg.WithAmbiguousNames = b
+	}
+}
+
 // OptWithBayes is an option that forces running bayes name-finding even when
 // the language is not supported by training sets.
 func OptWithBayes(b bool) Option {

ent/api/params.go

@@ -56,4 +56,8 @@ type FinderParams struct {
 	// WithAllMatches indicates that Verification results will return all
 	// found results, not only the BestResult.
 	WithAllMatches bool `json:"withAllMatches" form:"allMatches"`
+
+	// WithAmbiguousNames preserves detected ambigous uninomials like `America`
+	// or `Cancer`.
+	WithAmbiguousNames bool `json:"withAmbiguousNames" form:"ambiguousNames"`
 }

ent/heuristic/heuristic.go

@@ -29,15 +29,28 @@ func exploreNameCandidate(ts []token.TokenSN, d *dict.Dictionary) bool {
 
 	u := ts[0]
 
-	if u.Features().UninomialDict == dict.WhiteUninomial ||
-		(u.Indices().Species == 0 &&
-			u.Features().UninomialDict == dict.WhiteGenus) {
+	if u.Features().UninomialDict == dict.WhiteUninomial {
 		u.SetDecision(token.Uninomial)
 		return true
 	}
 
-	if u.Indices().Species == 0 ||
-		u.Features().UninomialDict == dict.BlackUninomial {
+	if u.Features().UninomialDict == dict.GreyUninomial {
+		u.SetDecision(token.PossibleUninomial)
+		return true
+	}
+
+	if u.Indices().Species == 0 {
+		if u.Features().UninomialDict == dict.WhiteGenus {
+			u.SetDecision(token.Uninomial)
+			return true
+		}
+		if u.Features().UninomialDict == dict.GreyGenus {
+			u.SetDecision(token.PossibleUninomial)
+			return true
+		}
+	}
+
+	if u.Features().UninomialDict == dict.BlackUninomial {
 		return false
 	}
 
@@ -53,7 +66,7 @@ func exploreNameCandidate(ts []token.TokenSN, d *dict.Dictionary) bool {
 	return true
 }
 
-func checkAsSpecies(t token.TokenSN, d *dict.Dictionary) bool {
+func checkAsSpecies(t token.TokenSN) bool {
 	if !t.Features().IsCapitalized &&
 		(t.Features().SpeciesDict == dict.WhiteSpecies ||
 			t.Features().SpeciesDict == dict.GreySpecies) {
@@ -65,12 +78,15 @@ func checkAsSpecies(t token.TokenSN, d *dict.Dictionary) bool {
 func checkAsGenusSpecies(ts []token.TokenSN, d *dict.Dictionary) bool {
 	g := ts[0]
 	s := ts[g.Indices().Species]
-
-	if !checkAsSpecies(s, d) {
+	if !checkAsSpecies(s) {
 		if g.Features().UninomialDict == dict.WhiteGenus {
 			g.SetDecision(token.Uninomial)
 			return true
 		}
+		if g.Features().UninomialDict == dict.GreyGenus {
+			g.SetDecision(token.PossibleUninomial)
+			return true
+		}
 		return false
 	}
 
@@ -126,7 +142,7 @@ func checkInfraspecies(ts []token.TokenSN, d *dict.Dictionary) {
 	s := ts[ts[0].Indices().Species]
 	isp := ts[i]
 
-	if checkGreyGeneraIsp(g, s, isp, d) || checkAsSpecies(ts[i], d) {
+	if checkGreyGeneraIsp(g, s, isp, d) || checkAsSpecies(ts[i]) {
 		ts[0].SetDecision(token.Trinomial)
 	}
 }

ent/output/output.go

@@ -40,20 +40,6 @@ type Meta struct {
 	// TotalSec is time spent for the whole process
 	TotalSec float32 `json:"totalSec"`
 
-	// WithBayes use of bayes during name-finding
-	WithBayes bool `json:"withBayes"`
-
-	// WithOddsAdjustment to adjust prior odds according to the dencity of
-	// scientific names in the text.
-	WithOddsAdjustment bool `json:"withOddsAdjustment"`
-
-	// WithPositionInBytes names get start/enc positionx in bytes
-	// instead of UTF-8 chars.
-	WithPositionInBytes bool `json:"withPositionInBytes"`
-
-	// WithVerification is true if results are checked by verification service.
-	WithVerification bool `json:"withVerification"`
-
 	// WordsAround shows the number of tokens preserved before and after
 	// a name-string candidate.
 	WordsAround int `json:"wordsAround"`
@@ -64,8 +50,33 @@ type Meta struct {
 	// LanguageDetected automatically for the text.
 	LanguageDetected string `json:"languageDetected,omitempty"`
 
+	// WithAllMatches is true if all verifcation results are shown.
+	WithAllMatches bool `json:"withAllMatches,omitempty"`
+
+	// WithAmbiguousNames is true if ambiguous uninomials are preserved.
+	// Examples of ambiguous uninomial names are `Cancer`, `America`.
+	WithAmbiguousNames bool `json:"withAmbiguousNames,omitempty"`
+
+	// WithUniqueNames is true when unique names are returned instead
+	// of every occurance of a name.
+	WithUniqueNames bool `json:"withUniqueNames,omitempty"`
+
+	// WithBayes use of bayes during name-finding
+	WithBayes bool `json:"withBayes,omitempty"`
+
+	// WithOddsAdjustment to adjust prior odds according to the dencity of
+	// scientific names in the text.
+	WithOddsAdjustment bool `json:"withOddsAdjustment,omitempty"`
+
+	// WithPositionInBytes names get start/enc positionx in bytes
+	// instead of UTF-8 chars.
+	WithPositionInBytes bool `json:"withPositionInBytes,omitempty"`
+
+	// WithVerification is true if results are checked by verification service.
+	WithVerification bool `json:"withVerification,omitempty"`
+
 	// WithLanguageDetection sets automatic language determination.
-	WithLanguageDetection bool `json:"withLanguageDetection"`
+	WithLanguageDetection bool `json:"withLanguageDetection,omitempty"`
 
 	// TotalWords is a number of 'normalized' words in the text
 	TotalWords int `json:"totalWords"`
@@ -115,11 +126,14 @@ type Name struct {
 	Cardinality int `json:"cardinality"`
 
 	// Verbatim shows name the way it was in the text.
-	Verbatim string `json:"verbatim"`
+	Verbatim string `json:"verbatim,omitempty"`
 
 	// Name is a normalized version of a name.
 	Name string `json:"name"`
 
+	// Decision about the quality of name detection.
+	Decision token.Decision `json:"-"`
+
 	// Odds show a probability that name detection was correct.
 	Odds float64 `json:"-"`
 
@@ -183,6 +197,7 @@ func postprocessNames(
 // newOutput is a constructor for Output type.
 func newOutput(
 	names []Name,
+	genera map[string]struct{},
 	ts []token.TokenSN,
 	version string,
 	cfg config.Config,
@@ -197,6 +212,9 @@ func newOutput(
 	meta := Meta{
 		Date:                time.Now(),
 		FinderVersion:       version,
+		WithAllMatches:      cfg.WithAllMatches,
+		WithAmbiguousNames:  cfg.WithAmbiguousNames,
+		WithUniqueNames:     cfg.WithUniqueNames,
 		WithBayes:           cfg.WithBayes,
 		WithOddsAdjustment:  cfg.WithOddsAdjustment,
 		WithVerification:    cfg.WithVerification,
@@ -207,6 +225,9 @@ func newOutput(
 		TotalNameCandidates: candidatesNum(ts),
 		TotalNames:          len(names),
 	}
+	if !cfg.WithAmbiguousNames {
+		names = FilterNames(names, genera)
+	}
 
 	if !cfg.WithBayesOddsDetails || cfg.WithOddsAdjustment {
 		postprocessNames(names, meta.TotalNameCandidates, cfg)
@@ -216,3 +237,16 @@ func newOutput(
 
 	return o
 }
+
+func FilterNames(names []Name, genera map[string]struct{}) []Name {
+	res := make([]Name, 0, len(names))
+	for i := range names {
+		if names[i].Decision == token.PossibleUninomial {
+			if _, ok := genera[names[i].Name]; !ok {
+				continue
+			}
+		}
+		res = append(res, names[i])
+	}
+	return res
+}