@@ -44,7 +44,7 @@ func tokenizeForNormalization(expr string) []tokenForNorm {
4444 word := current .String ()
4545 upper := strings .ToUpper (word )
4646 switch upper {
47- case "AND" , "OR" , "WITH" :
47+ case opAND , opOR , opWITH :
4848 tokens = append (tokens , tokenForNorm {value : upper , isOp : true })
4949 default :
5050 tokens = append (tokens , tokenForNorm {value : word })
@@ -76,109 +76,116 @@ func tokenizeForNormalization(expr string) []tokenForNorm {
7676 return tokens
7777}
7878
79- // normalizeTokens processes tokens and normalizes informal license names.
80- func normalizeTokens (tokens []tokenForNorm ) (string , error ) {
81- var result strings.Builder
82- var licenseWords []string
83- expectException := false // true if we just saw WITH
84-
85- flushLicense := func () error {
86- if len (licenseWords ) == 0 {
87- return nil
88- }
79+ // tokenNormalizer holds state for normalizing a stream of tokens.
80+ type tokenNormalizer struct {
81+ result strings.Builder
82+ licenseWords []string
83+ expectException bool
84+ }
8985
90- normalized , err := normalizeLicenseWords (licenseWords )
91- if err != nil {
92- return err
93- }
86+ func (n * tokenNormalizer ) flushPending () error {
87+ if n .expectException {
88+ return n .flushException ()
89+ }
90+ return n .flushLicense ()
91+ }
9492
95- if result .Len () > 0 && ! strings .HasSuffix (result .String (), "(" ) {
96- result .WriteString (" " )
97- }
98- result .WriteString (normalized )
99- licenseWords = nil
93+ func (n * tokenNormalizer ) flushLicense () error {
94+ if len (n .licenseWords ) == 0 {
10095 return nil
10196 }
10297
103- flushException := func () error {
104- if len ( licenseWords ) == 0 {
105- return nil
106- }
98+ normalized , err := normalizeLicenseWords ( n . licenseWords )
99+ if err != nil {
100+ return err
101+ }
107102
108- // Exception should be a single valid exception ID
109- exc := strings .Join (licenseWords , "-" )
110- if lookupException (exc ) == "" {
111- // Try the original form
112- exc = strings .Join (licenseWords , " " )
113- if lookupException (exc ) == "" {
114- return & LicenseError {License : exc , Err : ErrInvalidException }
115- }
116- }
103+ if n .result .Len () > 0 && ! strings .HasSuffix (n .result .String (), "(" ) {
104+ n .result .WriteString (" " )
105+ }
106+ n .result .WriteString (normalized )
107+ n .licenseWords = nil
108+ return nil
109+ }
117110
118- result .WriteString (" " )
119- result .WriteString (lookupException (exc ))
120- licenseWords = nil
111+ func (n * tokenNormalizer ) flushException () error {
112+ if len (n .licenseWords ) == 0 {
121113 return nil
122114 }
123115
124- for _ , tok := range tokens {
125- if tok .isOp {
126- if expectException {
127- if err := flushException (); err != nil {
128- return "" , err
129- }
130- expectException = false
131- } else {
132- if err := flushLicense (); err != nil {
133- return "" , err
134- }
135- }
136- result .WriteString (" " )
137- result .WriteString (tok .value )
138- if tok .value == "WITH" {
139- expectException = true
140- }
141- } else if tok .isParen {
142- if expectException {
143- if err := flushException (); err != nil {
144- return "" , err
145- }
146- expectException = false
147- } else {
148- if err := flushLicense (); err != nil {
149- return "" , err
150- }
151- }
152- if tok .value == "(" {
153- if result .Len () > 0 && ! strings .HasSuffix (result .String (), "(" ) && ! strings .HasSuffix (result .String (), " " ) {
154- result .WriteString (" " )
155- }
156- result .WriteString ("(" )
157- } else {
158- result .WriteString (")" )
159- }
160- } else if tok .isPlus {
161- // Plus attaches to previous license word
162- if len (licenseWords ) > 0 {
163- licenseWords [len (licenseWords )- 1 ] += "+"
164- }
165- } else {
166- // License word (or exception word if expectException)
167- licenseWords = append (licenseWords , tok .value )
116+ // Exception should be a single valid exception ID
117+ exc := strings .Join (n .licenseWords , "-" )
118+ if lookupException (exc ) == "" {
119+ // Try the original form
120+ exc = strings .Join (n .licenseWords , " " )
121+ if lookupException (exc ) == "" {
122+ return & LicenseError {License : exc , Err : ErrInvalidException }
168123 }
169124 }
170125
171- if expectException {
172- if err := flushException (); err != nil {
173- return "" , err
126+ n .result .WriteString (" " )
127+ n .result .WriteString (lookupException (exc ))
128+ n .licenseWords = nil
129+ return nil
130+ }
131+
132+ func (n * tokenNormalizer ) handleOp (tok tokenForNorm ) error {
133+ if err := n .flushPending (); err != nil {
134+ return err
135+ }
136+ n .expectException = false
137+ n .result .WriteString (" " )
138+ n .result .WriteString (tok .value )
139+ if tok .value == opWITH {
140+ n .expectException = true
141+ }
142+ return nil
143+ }
144+
145+ func (n * tokenNormalizer ) handleParen (tok tokenForNorm ) error {
146+ if err := n .flushPending (); err != nil {
147+ return err
148+ }
149+ n .expectException = false
150+ if tok .value == "(" {
151+ if n .result .Len () > 0 && ! strings .HasSuffix (n .result .String (), "(" ) && ! strings .HasSuffix (n .result .String (), " " ) {
152+ n .result .WriteString (" " )
174153 }
154+ n .result .WriteString ("(" )
175155 } else {
176- if err := flushLicense (); err != nil {
156+ n .result .WriteString (")" )
157+ }
158+ return nil
159+ }
160+
161+ // normalizeTokens processes tokens and normalizes informal license names.
162+ func normalizeTokens (tokens []tokenForNorm ) (string , error ) {
163+ n := & tokenNormalizer {}
164+
165+ for _ , tok := range tokens {
166+ var err error
167+ switch {
168+ case tok .isOp :
169+ err = n .handleOp (tok )
170+ case tok .isParen :
171+ err = n .handleParen (tok )
172+ case tok .isPlus :
173+ if len (n .licenseWords ) > 0 {
174+ n .licenseWords [len (n .licenseWords )- 1 ] += "+"
175+ }
176+ default :
177+ n .licenseWords = append (n .licenseWords , tok .value )
178+ }
179+ if err != nil {
177180 return "" , err
178181 }
179182 }
180183
181- return strings .TrimSpace (result .String ()), nil
184+ if err := n .flushPending (); err != nil {
185+ return "" , err
186+ }
187+
188+ return strings .TrimSpace (n .result .String ()), nil
182189}
183190
184191// normalizeLicenseWords takes a slice of words that should form a license name
0 commit comments