tools: add copyrighter for copyright modification

tools/copyrighter/CRContext.py

@@ -0,0 +1,108 @@
+#! /usr/bin/env bash
+#
+# Copyright (C) 2013-2017 Bilibili
+# Copyright (C) 2013-2017 Zhang Rui <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import datetime
+import os
+import fnmatch
+from sets import Set
+
+from copyrighter.CRCopyright import CRCopyright
+
+class CRContext:
+    file_ext_white_list = Set([
+        # '*',
+        'c',
+        'h',
+        'java',
+        'm',
+        'mk',
+        'py',
+        'sh',
+        ])
+
+    rel_path_black_list = Set([
+        '*.a',
+        '*.d',
+        '*.jar',
+        '*.lib',
+        '*.lproj',
+        '*.o',
+        '*.patch',
+        '*.pbxproj',
+        '*.pch',
+        '*.png',
+        '*.pyc',
+        '*.so',
+        '*.strings',
+        '*.xcassets',
+        '*.xcodeproj',
+        '*.xcuserdatad',
+        '*.xcworkspace',
+        '*.xib',
+
+        'android/contrib',
+        'android/ijkplayer/build',
+        'android/ijkplayer/*/build',
+        'android/ijkplayer/*/libs',
+        'android/ijkplayer/*/obj',
+        'android/ijkplayer/*/res',
+        'android/patches',
+        'extra',
+        'ijkmedia/ijkyuv',
+        'ijkmedia/ijkj4a',
+        'ijkprof/android-ndk-profiler',
+        'ios/build',
+        'ios/contrib',
+        'ios/ffmpeg-*',
+        'ios/openssl-*',
+        ])
+
+    def __init__(self, verbose = True, dryrun = True):
+        self.verbose    = verbose
+        self.dryrun     = dryrun
+        self.root_path  = os.path.realpath(os.path.join(__file__, '../../..'))
+        self.year       = datetime.date.today().year
+
+    def get_path_of_file(self, file):
+        if not os.path.isabs(file):
+            file = os.path.realpath(os.path.join(self.root_path, file))
+        return file
+
+    def get_relpath(self, file):
+        return os.path.relpath(file, self.root_path)
+
+    def is_black_path(self, path):
+        rel_path = self.get_relpath(path)
+        for black_item in CRContext.rel_path_black_list:
+            if fnmatch.fnmatch(rel_path, black_item):
+                # print '  + fnmatch %s %s' % (rel_path, black_item)
+                return True
+            # else:
+                # print '  - fnmatch %s %s' % (rel_path, black_item)
+        return False
+
+    def need_copyright(self, file):
+        if '*' in CRContext.file_ext_white_list:
+            return True
+        ext = os.path.splitext(file)[1][1:]
+        return ext in CRContext.file_ext_white_list
+
+    def log_file(self, tag, file):
+        if self.verbose:
+            rel_path = self.get_relpath(file)
+            print '%-10s %s' % (tag, rel_path)

tools/copyrighter/CRCopyright.py

@@ -0,0 +1,95 @@
+#! /usr/bin/env bash
+#
+# Copyright (C) 2013-2017 Bilibili
+# Copyright (C) 2013-2017 Zhang Rui <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import re
+
+class CRCopyright:
+    regexp = None
+
+    def __init__(self, name, url = None, prefix = None, symbol = None, years = None):
+        self.name   = name
+        self.url    = url
+        self.prefix = prefix
+        self.symbol = symbol
+        self.years  = years
+        assert prefix
+        assert symbol
+        assert years
+        assert name
+
+    def duplicate(self):
+        return CRCopyright(
+            name   = self.name,
+            url    = self.url,
+            prefix = self.prefix,
+            symbol = self.symbol,
+            years  = self.years,
+            )
+
+    def get_line(self):
+        line = '%sCopyright %s %s %s' % (self.prefix, self.symbol, self.years, self.name)
+        if self.url:
+            line += ' <%s>' % self.url
+        return line
+
+    def match_name(self, name):
+        if not self.name or not name:
+            return False
+        return self.name.lower() == name.lower()
+
+    def match_name(self, url):
+        if not self.url or not url:
+            return False
+        return self.url.lower() == url.lower()
+
+    @staticmethod
+    def scan_line(context, line):
+        if line[-1] == '\n':
+            line = line[0:-1]
+        if not CRCopyright.regexp:
+            # CRCopyright.regexp = re.compile("^([#\*\/\s]+)Copyright(?:\s+)([\(\)\w]+)(?:\s+)([\d]+|[\d]+-[\d]+)(?:\s+)([\w\s]+)[$|(<[\w\.@]+>)$]")
+            # 1 prefix
+            # 3 symbol
+            # 5 year
+            # 6 name
+            # 7 url
+            CRCopyright.regexp = re.compile(
+                pattern = "^([#\*\/\s]+)Copyright(\s+)([\(\)\w]+)(\s+)([0-9\-]+)([^<]+)(.*)$",
+                flags = re.IGNORECASE,
+                )
+        m = CRCopyright.regexp.match(line)
+        if m:
+            prefix = m.group(1)
+            symbol = m.group(3).strip()
+            years  = m.group(5).strip()
+            name   = m.group(6).strip()
+            url    = m.group(7).strip().strip('<>')
+            if prefix and symbol and years and name:
+                cr = CRCopyright(
+                    prefix = prefix,
+                    symbol = symbol,
+                    years  = years,
+                    name   = name,
+                    url    = url,
+                    )
+                return cr
+            # print '>====='
+            # for i in range(1, 8):
+            #     print '> %d "%s"' % (i, match.group(i))
+            # print '>====='
+        return None

tools/copyrighter/CRFile.py

@@ -0,0 +1,123 @@
+#! /usr/bin/env bash
+#
+# Copyright (C) 2013-2017 Bilibili
+# Copyright (C) 2013-2017 Zhang Rui <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import shutil
+
+from copyrighter.CRCopyright import CRCopyright
+
+class CRFile:
+    def __init__(self, context, file):
+        self.dirty           = False
+        self.context         = context
+        self.abs_path        = context.get_path_of_file(file)
+        self.file_ext        = os.path.splitext(self.abs_path)[1][1:]
+        self.copyright_names = {}
+        self.copyright_urls  = {}
+        self.need_insert_bilibili_copyright = True
+
+    def update(self):
+        if not self.dirty:
+            self.context.log_file('~ remain', self.abs_path)
+            return
+        tmp_path = self.abs_path + '.tmp'
+        if self.context.dryrun:
+            src = open(self.abs_path, 'r')
+        else:
+            shutil.copy2(self.abs_path, tmp_path)
+            src = open(tmp_path, 'r')
+            tmp = open(self.abs_path, 'w')
+        did_insert_bilibili_copyright = False
+        for line in src:
+            if self.need_insert_bilibili_copyright and not did_insert_bilibili_copyright:
+                copyright = CRCopyright.scan_line(self.context, line)
+                if copyright:
+                    copyright.name = 'Bilibili'
+                    copyright.url  = None
+                    if not self.context.dryrun:
+                        tmp.write(copyright.get_line())
+                        tmp.write("\n")
+                    # print '    insert %s' % copyright.get_line()
+                    did_insert_bilibili_copyright = True
+            if not self.context.dryrun:
+                tmp.write(line)    
+        src.close()
+        if not self.context.dryrun:
+            tmp.close()
+            os.remove(tmp_path)
+
+        if self.need_insert_bilibili_copyright and did_insert_bilibili_copyright:
+            self.context.log_file('+ update', self.abs_path)
+        else:
+            self.context.log_file('~ missing', self.abs_path)
+
+    def copyright_names(self):
+        return self.copyright_names.keys()
+
+    def copyright_urls(self):
+        return self.copyright_urls.keys()
+
+    def __parse_line(self, line):
+        copyright = CRCopyright.scan_line(self.context, line)
+        if copyright:
+            # print "match %s" % copyright.name
+            self.copyright_names[copyright.name.lower()] = copyright
+            self.copyright_urls[copyright.url.lower()]   = copyright
+        return True
+
+    @staticmethod
+    def load_from_file(context, file):
+        parsed_lines = 0;
+        crf          = CRFile(context = context, file = file)
+        f            = open(crf.abs_path, 'r')
+        for line in f:
+            if parsed_lines > 20:
+                break
+            parsed_lines += 1
+            crf.__parse_line(line)
+        f.close()
+        # TODO: use a visitor
+        if 'bilibili' not in crf.copyright_names:
+            if 'zhang rui' in crf.copyright_names or '[email protected]' in crf.copyright_urls:
+                crf.need_insert_bilibili_copyright = True
+                crf.dirty                          = True
+        return crf
+
+    @staticmethod
+    def update_path(context, file):
+        base_name = os.path.basename(file)
+        abs_path  = context.get_path_of_file(file)
+        if   base_name.startswith('.'):
+            context.log_file('- hidden', abs_path)
+            return
+        elif context.is_black_path(abs_path):
+            context.log_file('- black', abs_path)
+            return
+        elif os.path.islink(abs_path):
+            context.log_file('- link', abs_path)
+            return
+        elif os.path.isdir(abs_path):
+            for sub_file in os.listdir(abs_path):
+                sub_path = os.path.realpath(os.path.join(abs_path, sub_file))
+                CRFile.update_path(context, sub_path)
+        elif not context.need_copyright(abs_path):
+            context.log_file('- nohead', abs_path)
+            return
+        elif os.path.isfile(abs_path):
+            src_file = CRFile.load_from_file(context, abs_path)
+            src_file.update()

tools/copyrighter/__init__.py

@@ -0,0 +1,17 @@
+#! /usr/bin/env bash
+#
+# Copyright (C) 2013-2017 Bilibili
+# Copyright (C) 2013-2017 Zhang Rui <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

tools/copyrighter/__main__.py

@@ -0,0 +1,40 @@
+#! /usr/bin/env bash
+#
+# Copyright (C) 2013-2017 Bilibili
+# Copyright (C) 2013-2017 Zhang Rui <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+import os
+import fnmatch
+from sets import Set
+
+if not __package__:
+    path = os.path.join(os.path.dirname(__file__), os.pardir)
+    sys.path.insert(0, path)
+
+from copyrighter.CRContext import CRContext
+from copyrighter.CRFile import CRFile
+
+context = CRContext();
+
+target_list = Set([
+    'ijkmedia',
+    'extra/init-extra.sh'
+    ])
+
+for target in Set(['']):
+# for target in target_list:
+    CRFile.update_path(context, target)