main.go

/*
Copyright 2020 The Kubermatic Kubernetes Platform contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/sirupsen/logrus"
)

// These variables get set by ldflags during compilation.
var (
	BuildTag    string
	BuildCommit string
	BuildDate   string // RFC3339 format ("2006-01-02T15:04:05Z07:00")
)

// regular expressions
var (
	now       = time.Now()
	thisYear  = now.Year()
	startYear = 2014

	// Search for "YEAR" which exists in the boilerplate,
	// but shouldn't in the real thing.
	yearRe = regexp.MustCompile("YEAR")

	// finds all dates from startYear to the current Year.
	dateRe = regexp.MustCompile(getDateRegex())

	// detect generated files by presence iof this string in the first non-stripped line
	generatedRe = regexp.MustCompile("(been generated|generated by|do not edit)")

	// strip "// +build \n\n" and "//go:build \n\n" build constraints
	goBuildConstraintsRe = regexp.MustCompile(`^(//(go:| \+)build.*\n)+\n*`)

	// strip #!.* from shell scripts
	shebangRe = regexp.MustCompile(`^(#!.*\n)\n*`)
)

// runtime stats
var (
	checkedFiles int
	failedFiles  int
	ignoredFiles int
)

type stringSlice []string

func (s *stringSlice) String() string {
	return fmt.Sprint(*s)
}

func (s *stringSlice) Set(value string) error {
	if _, err := filepath.Match(value, "test"); err != nil {
		return fmt.Errorf("invalid glob pattern: %w", err)
	}

	*s = append(*s, value)
	return nil
}

// flags
var (
	boilerplateDir string
	verbose        bool
	version        bool
	excludes       stringSlice = []string{
		".git",
		"vendor",
		"_build",
		"zz_generated.*",
		"zz_generated_*",
	}
)

func main() {
	flag.StringVar(&boilerplateDir, "boilerplates", "hack/boilerplate/", "Directory containing the boilerplate files for file extensions.")
	flag.Var(&excludes, "exclude", fmt.Sprintf("glob expressions (relative to cwd) to exclude, can be given multiple times (%v are excluded by default)", excludes))
	flag.BoolVar(&verbose, "verbose", false, "give verbose output regarding why a file does not pass.")
	flag.BoolVar(&version, "version", false, "show application version and exit immediately.")
	flag.Parse()

	if version {
		commit := "dev"
		if BuildCommit != "" {
			commit = BuildCommit[:10]
		}

		fmt.Printf("boilerplate %s (%s), built with %s on %s\n", BuildTag, commit, runtime.Version(), BuildDate)
		return
	}

	log := logrus.New()
	log.SetFormatter(&logrus.TextFormatter{
		FullTimestamp:   true,
		TimestampFormat: time.RFC1123,
	})

	if verbose {
		log.SetLevel(logrus.DebugLevel)
	}

	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to determine working directory: %v", err)
	}

	boilerplates, err := getBoilerplateForExtensions(boilerplateDir)
	if err != nil {
		log.Fatalf("Failed to read boilerplates: %v", err)
	}

	sources := flag.Args()
	if len(sources) == 0 {
		sources = []string{cwd}
	}

	for idx, source := range sources {
		abs, err := filepath.Abs(source)
		if err != nil {
			log.Fatalf("Failed to determine absolute path for %q: %v", source, err)
		}

		sources[idx] = abs
	}

	success := true

	for _, source := range sources {
		ok, err := processSource(cwd, source, log, boilerplates, excludes)
		if err != nil {
			log.Fatalf("Failed to process %q: %v", source, err)
		}

		if !ok {
			success = false
		}
	}

	if success {
		log.Infof("All licenses match, checked %d files, ignored %d.", checkedFiles, ignoredFiles)
	} else {
		verb := "are"
		if failedFiles == 1 {
			verb = "is"
		}

		log.Errorf("%d out of %d checked files %s invalid.", failedFiles, checkedFiles, verb)
		os.Exit(1)
	}
}

func processSource(cwd string, source string, log logrus.FieldLogger, boilerplates map[string]string, excludes []string) (bool, error) {
	success := true

	err := filepath.Walk(source, func(currentPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		ext := fileExtension(currentPath)
		base := path.Base(currentPath)

		rel, err := filepath.Rel(cwd, currentPath)
		if err != nil {
			return fmt.Errorf("failed to determine relative path of %q: %w", currentPath, err)
		}

		for _, exclude := range excludes {
			excluded, _ := filepath.Match(exclude, base)
			if !excluded {
				excluded, _ = filepath.Match(exclude, rel)
			}

			if excluded {
				ignoredFiles++

				if info.IsDir() {
					return filepath.SkipDir
				}

				return nil
			}
		}

		// only process files
		if info.IsDir() {
			return nil
		}

		// Skip files that we don't have a boilerplate for
		_, extFound := boilerplates[ext]
		_, baseFound := boilerplates[base]
		if !extFound && !baseFound {
			return nil
		}

		ok, err := processFile(currentPath, log.WithField("file", rel), boilerplates)
		if err != nil {
			return fmt.Errorf("failed to process %q: %w", rel, err)
		}

		if !ok {
			success = false
		}

		return nil
	})

	return success, err
}

func processFile(filename string, log logrus.FieldLogger, boilerplates map[string]string) (bool, error) {
	fileContent, err := os.ReadFile(filename)
	if err != nil {
		return false, fmt.Errorf("failed to read file: %w", err)
	}

	// determine extension to use to lookup the boilerplate header
	extension := fileExtension(filename)
	if extension == "" {
		extension = path.Base(filename)
	}
	boilerplate, ok := boilerplates[extension]
	if !ok {
		return false, fmt.Errorf("no boilerplate registered for extension %q", extension)
	}

	// remove extra content from the top of files
	switch extension {
	case "go":
		fileContent = goBuildConstraintsRe.ReplaceAll(fileContent, nil)
	case "sh", "py":
		fileContent = shebangRe.ReplaceAll(fileContent, nil)
	}

	if isGenerated(fileContent) {
		ignoredFiles++
		log.Debug("Ignoring generated file.")
		return true, nil
	}

	checkedFiles++

	// if our test file is smaller than the reference it surely fails!
	if len(fileContent) < len(boilerplate) {
		failedFiles++
		log.Warn("File is smaller than its boilerplate.")
		return false, nil
	}

	// trim the file to the same length as our boilerplate header
	fileContent = fileContent[0:len(boilerplate)]

	// is YEAR in the file
	if yearRe.Match(fileContent) {
		failedFiles++
		log.Warn("Boilerplate contains YEAR placeholder, but should not.")
		return false, nil
	}

	// replace the actual year eg. "2014" with "YEAR" to compare with the boilerplate file
	fileContent = dateRe.ReplaceAll(fileContent, []byte("YEAR"))

	// check if the file header matches the boilerplate
	actual := strings.TrimSpace(string(fileContent))
	expected := strings.TrimSpace(boilerplate)
	if actual != expected {
		failedFiles++

		log.Warn("Invalid boilerplate.")
		log.Debugf("Expected: %q", expected)
		log.Debugf("Actual: %q", actual)

		return false, nil
	}

	log.Debug("File is valid.")

	return true, nil
}

func isGenerated(content []byte) bool {
	lines := bytes.SplitN(content, []byte{'\n'}, 2)

	return generatedRe.Match(bytes.ToLower(lines[0]))
}

func fileExtension(filename string) string {
	base := path.Base(filename)
	i := strings.LastIndex(base, ".")
	if i < 0 {
		// no dot found in the filename
		return ""
	}

	return base[i+1:]
}

// getBoilerplateForExtensions reads the boilerplate.*.txt files in the directory
// and returns a map of file extension to the files content
func getBoilerplateForExtensions(sourceDir string) (map[string]string, error) {
	boilerplate := map[string]string{}

	matches, err := filepath.Glob(path.Join(sourceDir, "boilerplate.*.txt"))
	if err != nil {
		return nil, fmt.Errorf("finding files via glob: %w", err)
	}

	for _, match := range matches {
		parts := strings.Split(path.Base(match), ".")
		if len(parts) != 3 {
			return nil, fmt.Errorf("wrong filename for boilerplate file: %q must be \"boilerplate.EXTENSION.txt\"", match)
		}

		content, err := os.ReadFile(match)
		if err != nil {
			return nil, fmt.Errorf("reading file: %w", err)
		}

		// map file extension to the boilerplate for the file
		boilerplate[parts[1]] = string(content)
	}

	return boilerplate, nil
}

// getDateRegex returns a regex like "(2014|2015|2016|2017|2018)"
// containing all years from 2014 until the current year.
func getDateRegex() string {
	var dates []string
	for i := startYear; i <= thisYear; i++ {
		dates = append(dates, strconv.Itoa(i))
	}
	return "(" + strings.Join(dates, "|") + ")"
}