diff --git a/Utilities/StaticAnalyzers/scripts/callgraph.py b/Utilities/StaticAnalyzers/scripts/callgraph.py index 9cf6f8083d13b..1b5c248e633a9 100755 --- a/Utilities/StaticAnalyzers/scripts/callgraph.py +++ b/Utilities/StaticAnalyzers/scripts/callgraph.py @@ -1,81 +1,83 @@ #! /usr/bin/env python3 from __future__ import print_function +import networkx as nx import re import yaml topfunc = re.compile(r"::(accumulate|acquire|startingNewLoop|duringLoop|endOfLoop|beginOfJob|endOfJob|produce|analyze|filter|beginLuminosityBlock|beginRun|beginStream|streamBeginRun|streamBeginLuminosityBlock|streamEndRun|streamEndLuminosityBlock|globalBeginRun|globalEndRun|globalBeginLuminosityBlock|globalEndLuminosityBlock|endRun|endLuminosityBlock)\(") -baseclass = re.compile(r"edm::(one::|stream::|global::)?(ED(Producer|Filter|Analyzer|(IterateNTimes|NavigateEvents)?Looper)(Base)?|impl::(ExternalWork|Accumulator))") +baseclass = re.compile( + r"edm::(one::|stream::|global::)?(ED(Producer|Filter|Analyzer|(IterateNTimes|NavigateEvents)?Looper)(Base)?|impl::(ExternalWork|Accumulator))") farg = re.compile(r"\(.*?\)") tmpl = re.compile(r'<.*?>') toplevelfuncs = set() epfuncre = re.compile(r"edm::eventsetup::EventSetupRecord::get<.*>\(.*\)") skipfunc = re.compile(r"TGraph::IsA\(.*\)") -epfuncs=set() +epfuncs = set() -import networkx as nx -G=nx.DiGraph() +G = nx.DiGraph() -#g = open('module_to_package.txt') -#module2package=dict() -#for line in g: -# fields = line.strip().split(';') -# if len(fields) <2: -# continue -# module2package.setdefault(fields[1], []).append(fields[0]) +# g = open('module_to_package.txt') +# module2package=dict() +# for line in g: +# fields = line.strip().split(';') +# if len(fields) <2: +# continue +# module2package.setdefault(fields[1], []).append(fields[0]) # -#i = open('module_to_package.yaml', 'w') -#yaml.dump(module2package, i) -#i.close() +# i = open('module_to_package.yaml', 'w') +# yaml.dump(module2package, i) +# i.close() h = open('module_to_package.yaml', 'r') -module2package=yaml.load(h, Loader=yaml.FullLoader) +module2package = yaml.load(h, Loader=yaml.FullLoader) with open('function-calls-db.txt') as f: - for line in f : - fields = line.split("'") + for line in f: + fields = line.split("'") if len(fields) < 3: continue - if fields[2] == ' calls function ' : - if not skipfunc.search(line) : - G.add_edge(fields[1],fields[3],kind=fields[2]) - if epfuncre.search(fields[3]) : - epfuncs.add(fields[3]) - if fields[2] == ' overrides function ' : - if baseclass.search(fields[3]) : - if topfunc.search(fields[3]) : - toplevelfuncs.add(fields[1]) - G.add_edge(fields[1],fields[3],kind=' overrides function ') - else : - if not skipfunc.search(line) : - G.add_edge(fields[3],fields[1],kind=' calls override function ') - if epfuncre.search(fields[1]) : epfuncs.add(fields[1]) - - + if fields[2] == ' calls function ': + if not skipfunc.search(line): + G.add_edge(fields[1], fields[3], kind=fields[2]) + if epfuncre.search(fields[3]): + epfuncs.add(fields[3]) + if fields[2] == ' overrides function ': + if baseclass.search(fields[3]): + if topfunc.search(fields[3]): + toplevelfuncs.add(fields[1]) + G.add_edge(fields[1], fields[3], kind=' overrides function ') + else: + if not skipfunc.search(line): + G.add_edge(fields[3], fields[1], + kind=' calls override function ') + if epfuncre.search(fields[1]): + epfuncs.add(fields[1]) -callstacks=set() +callstacks = set() for tfunc in toplevelfuncs: - for epfunc in epfuncs: - if G.has_node(tfunc) and G.has_node(epfunc) and nx.has_path(G,tfunc,epfunc) : - path = nx.shortest_path(G,tfunc,epfunc) - cs=str("") - previous=str("") - for p in path : - if epfuncre.search(p): break - stripped=re.sub(farg,"()",p) - if previous != stripped: - cs+=' '+stripped+";" - previous = stripped - callstacks.add(cs) - break + for epfunc in epfuncs: + if G.has_node(tfunc) and G.has_node(epfunc) and nx.has_path(G, tfunc, epfunc): + path = nx.shortest_path(G, tfunc, epfunc) + cs = str("") + previous = str("") + for p in path: + if epfuncre.search(p): + break + stripped = re.sub(farg, "()", p) + if previous != stripped: + cs += ' ' + stripped + ";" + previous = stripped + callstacks.add(cs) + break -report=dict() +report = dict() for key in sorted(module2package.keys()): - for value in sorted(module2package[key]): - vre=re.compile(' %s::.*();' % value) - for cs in sorted(callstacks): - if vre.search(cs): - report.setdefault(key, {}).setdefault(value, []).append(cs) -r=open('eventsetuprecord-get.yaml', 'w') -yaml.dump(report,r) + for value in sorted(module2package[key]): + vre = re.compile(' %s::.*();' % value) + for cs in sorted(callstacks): + if vre.search(cs): + report.setdefault(key, {}).setdefault(value, []).append(cs) +r = open('eventsetuprecord-get.yaml', 'w') +yaml.dump(report, r) diff --git a/Utilities/StaticAnalyzers/scripts/class-composition.py b/Utilities/StaticAnalyzers/scripts/class-composition.py index 9e358a8290527..5256b6497a19a 100755 --- a/Utilities/StaticAnalyzers/scripts/class-composition.py +++ b/Utilities/StaticAnalyzers/scripts/class-composition.py @@ -1,16 +1,20 @@ #! /usr/bin/env python3 from __future__ import print_function +import networkx as nx import re stdcl = re.compile("^std::(.*)[^>]$") stdptr = re.compile("(.*)_ptr$") datacl = re.compile("^class ") bfunc = re.compile("^function ") mbcl = re.compile("(base|data) class") -farg = re.compile("(.*)\(\w+\)") -nsep = re.compile("\:\:") -topfunc = re.compile("::(produce|analyze|filter|beginLuminosityBlock|beginRun|beginStream)\(") -baseclass = re.compile("edm::(one::|stream::|global::)?ED(Producer|Filter|Analyzer)(Base)?") -getfunc = re.compile("edm::eventsetup::EventSetupRecord::get\<.*\>\((.*)&\) const") +farg = re.compile(r"(.*)\(\w+\)") +nsep = re.compile(r"\:\:") +topfunc = re.compile( + r"::(produce|analyze|filter|beginLuminosityBlock|beginRun|beginStream)\(") +baseclass = re.compile( + "edm::(one::|stream::|global::)?ED(Producer|Filter|Analyzer)(Base)?") +getfunc = re.compile( + r"edm::eventsetup::EventSetupRecord::get\<.*\>\((.*)&\) const") handle = re.compile("(.*),?class edm::ES(.*)Handle<(.*)>") statics = set() toplevelfuncs = set() @@ -26,82 +30,88 @@ memberclasses = set() derivedclasses = set() -import networkx as nx -G=nx.DiGraph() -H=nx.DiGraph() -I=nx.DiGraph() +Gdg = nx.DiGraph() +Hdg = nx.DiGraph() +Idg = nx.DiGraph() f = open('classes.txt.dumperall') -for line in f : - if mbcl.search(line) : - fields = line.split("'") - if fields[2] == ' member data class ': - if not stdcl.search(fields[2]) : H.add_edge(fields[1],fields[3],kind=fields[2]) - if fields[2] == ' templated member data class ': - H.add_edge(fields[1],fields[5],kind=fields[3]) - if fields[2] == ' base class ': - H.add_edge(fields[1],fields[3],kind=fields[2]) - I.add_edge(fields[3],fields[1],kind=' derived class') +for line in f: + if mbcl.search(line): + fields = line.split("'") + if fields[2] == ' member data class ': + if not stdcl.search(fields[2]): + Hdg.add_edge(fields[1], fields[3], kind=fields[2]) + if fields[2] == ' templated member data class ': + Hdg.add_edge(fields[1], fields[5], kind=fields[3]) + if fields[2] == ' base class ': + Hdg.add_edge(fields[1], fields[3], kind=fields[2]) + Idg.add_edge(fields[3], fields[1], kind=' derived class') f.close() f = open('function-calls-db.txt') -for line in f : - if not bfunc.search(line) : continue - fields = line.split("'") - if fields[2] == ' calls function ' : - G.add_edge(fields[1],fields[3],kind=' calls function ') - if getfunc.search(fields[3]) : - dataclassfuncs.add(fields[3]) - m = getfunc.match(fields[3]) - n = handle.match(m.group(1)) - if n : o = n.group(3) - else : o = m.group(1) - p = re.sub("class ","",o) - dataclass = re.sub("struct ","",p) - dataclasses.add(dataclass) - if fields[2] == ' overrides function ' : - if baseclass.search(fields[3]) : - G.add_edge(fields[1],fields[3],kind=' overrides function ') - if topfunc.search(fields[3]) : toplevelfuncs.add(fields[1]) - else : G.add_edge(fields[3],fields[1], kind=' calls override function ') - if fields[2] == ' static variable ' : - G.add_edge(fields[1],fields[3],kind=' static variable ') - statics.add(fields[3]) +for line in f: + if not bfunc.search(line): + continue + fields = line.split("'") + if fields[2] == ' calls function ': + Gdg.add_edge(fields[1], fields[3], kind=' calls function ') + if getfunc.search(fields[3]): + dataclassfuncs.add(fields[3]) + m = getfunc.match(fields[3]) + n = handle.match(m.group(1)) + if n: + o = n.group(3) + else: + o = m.group(1) + p = re.sub("class ", "", o) + dataclass = re.sub("struct ", "", p) + dataclasses.add(dataclass) + if fields[2] == ' overrides function ': + if baseclass.search(fields[3]): + Gdg.add_edge(fields[1], fields[3], kind=' overrides function ') + if topfunc.search(fields[3]): + toplevelfuncs.add(fields[1]) + else: + Gdg.add_edge(fields[3], fields[1], kind=' calls override function ') + if fields[2] == ' static variable ': + Gdg.add_edge(fields[1], fields[3], kind=' static variable ') + statics.add(fields[3]) f.close() visited = set() nodes = sorted(dataclasses) for node in nodes: - if node in visited: - continue - visited.add(node) - if node in H : stack = [(node,iter(H[node]))] - if node in I : - Q=nx.dfs_preorder_nodes(I,node) - for q in Q: - print("class '"+q+"'") - if q in H : - stack.append( ( q, iter( H[q] ) ) ) - while stack: - parent,children = stack[-1] - print("class '"+parent+"'") - try: - child = next(children) - if child not in visited: - visited.add(child) - if not stdcl.search(child): - print("class '"+child+"'") - stack.append( ( child, iter( H[child] ) ) ) - kind=H[parent][child]['kind'] - print(parent, kind, child) - if stdptr.search(kind): - if child in I : - Q=nx.dfs_preorder_nodes(I,child) - for q in Q : - print("class '"+q+"'") - if q in H : - stack.append( ( q, iter( H[q] ) ) ) - except StopIteration: - stack.pop() + if node in visited: + continue + visited.add(node) + if node in Hdg: + stack = [(node, iter(Hdg[node]))] + if node in Idg: + Qdg = nx.dfs_preorder_nodes(Idg, node) + for q in Qdg: + print("class '"+q+"'") + if q in Hdg: + stack.append((q, iter(Hdg[q]))) + while stack: + parent, children = stack[-1] + print("class '"+parent+"'") + try: + child = next(children) + if child not in visited: + visited.add(child) + if not stdcl.search(child): + print("class '"+child+"'") + stack.append((child, iter(Hdg[child]))) + kind = Hdg[parent][child]['kind'] + print(parent, kind, child) + if stdptr.search(kind): + if child in Idg: + Qdg = nx.dfs_preorder_nodes(Idg, child) + for q in Qdg: + print("class '"+q+"'") + if q in Hdg: + stack.append((q, iter(Hdg[q]))) + except StopIteration: + stack.pop() diff --git a/Utilities/StaticAnalyzers/scripts/classnames-extract.py b/Utilities/StaticAnalyzers/scripts/classnames-extract.py index 85bf6dd810d81..e809d58429d6c 100755 --- a/Utilities/StaticAnalyzers/scripts/classnames-extract.py +++ b/Utilities/StaticAnalyzers/scripts/classnames-extract.py @@ -1,11 +1,11 @@ #! /usr/bin/env python3 -f = open('classes.txt','r') -g = open('classnames.txt','w') -for line in f : - fields = line.split("'") - if fields[0] == 'class ' : - g.write(fields[1]+'\n') +f = open('classes.txt', 'r') +g = open('classnames.txt', 'w') +for line in f: + fields = line.split("'") + if fields[0] == 'class ': + g.write(fields[1]+'\n') f.close() g.close() diff --git a/Utilities/StaticAnalyzers/scripts/create_statics_esd_reports.sh b/Utilities/StaticAnalyzers/scripts/create_statics_esd_reports.sh index 92f6c6a81df0f..2f00100625e32 100755 --- a/Utilities/StaticAnalyzers/scripts/create_statics_esd_reports.sh +++ b/Utilities/StaticAnalyzers/scripts/create_statics_esd_reports.sh @@ -48,7 +48,7 @@ sort -u edm-global-classes.txt.unsorted | grep -v -e"^EDM global class " >edm-gl if [ ! -f ./callgraph.py ] then cp -pv ${CMSSW_BASE}/src/Utilities/StaticAnalyzers/scripts/callgraph.py . - cp -pv ${CMSSW_BASE}/src/Utilities/StaticAnalyzers/scripts/modules_to_package.yaml . + cp -pv ${CMSSW_BASE}/src/Utilities/StaticAnalyzers/scripts/module_to_package.yaml . fi touch eventsetuprecord-get-all.txt eventsetuprecord-get.txt ./callgraph.py 2>&1 | tee eventsetuprecord-get.txt diff --git a/Utilities/StaticAnalyzers/scripts/data-class-funcs.py b/Utilities/StaticAnalyzers/scripts/data-class-funcs.py index 06758a1cff2d9..2fb8581cf96b7 100755 --- a/Utilities/StaticAnalyzers/scripts/data-class-funcs.py +++ b/Utilities/StaticAnalyzers/scripts/data-class-funcs.py @@ -1,16 +1,22 @@ #! /usr/bin/env python3 from __future__ import print_function +import fileinput +import networkx as nx import re datacl = re.compile("^class ") bfunc = re.compile("^function ") mbcl = re.compile("(base|data) class") -farg = re.compile("(.*)\(\w+\)") -nsep = re.compile("\:\:") -topfunc = re.compile("::(produce|analyze|filter|beginLuminosityBlock|beginRun|beginStream)\(") -baseclass = re.compile("edm::(one::|stream::|global::)?ED(Producer|Filter|Analyzer)(Base)?") -getfunc = re.compile("edm::eventsetup::EventSetupRecord::get\<.*\>\((.*)&\) const") +farg = re.compile(r"(.*)\(\w+\)") +nsep = re.compile(r"\:\:") +topfunc = re.compile( + r"::(produce|analyze|filter|beginLuminosityBlock|beginRun|beginStream)\(") +baseclass = re.compile( + r"edm::(one::|stream::|global::)?ED(Producer|Filter|Analyzer)(Base)?") +getfunc = re.compile( + r"edm::eventsetup::EventSetupRecord::get\<.*\>\((.*)&\) const") handle = re.compile("(.*),?class edm::ES(.*)Handle<(.*)>") -skip = re.compile("edm::serviceregistry::ServicesManager::MakerHolder::add() const") +skip = re.compile( + "edm::serviceregistry::ServicesManager::MakerHolder::add() const") statics = set() toplevelfuncs = set() onefuncs = set() @@ -23,171 +29,182 @@ dclasses = set() dataclasses = set() flaggedclasses = set() -import networkx as nx -G=nx.DiGraph() -H=nx.DiGraph() +G = nx.DiGraph() +H = nx.DiGraph() f = open('classes.txt.dumperft') for line in f: - if datacl.search(line) : - classname = line.split("'")[1] - dclasses.add(classname) + if datacl.search(line): + classname = line.split("'")[1] + dclasses.add(classname) f.close() f = open('classes.txt.inherits') for line in f: - if datacl.search(line) : - classname = line.split("'")[1] - dataclasses.add(classname) + if datacl.search(line): + classname = line.split("'")[1] + dataclasses.add(classname) f.close() f = open('class-checker.txt') for line in f: - if mbcl.search(line): - fields = line.split("'") - classname = fields[1] - funcname = fields[3] - badclasses.add(classname) - badfuncs.add(funcname) + if mbcl.search(line): + fields = line.split("'") + classname = fields[1] + funcname = fields[3] + badclasses.add(classname) + badfuncs.add(funcname) f.close() f = open('classes.txt.dumperall') -for line in f : - if mbcl.search(line) : - fields = line.split("'") - if fields[2] == ' member data class ': - H.add_edge(fields[1],fields[3],kind=fields[2]) - if fields[2] == ' templated member data class ': - H.add_edge(fields[1],fields[3],kind=fields[2]) - if fields[2] == ' base class ': - H.add_edge(fields[1],fields[3],kind=fields[2]) +for line in f: + if mbcl.search(line): + fields = line.split("'") + if fields[2] == ' member data class ': + H.add_edge(fields[1], fields[3], kind=fields[2]) + if fields[2] == ' templated member data class ': + H.add_edge(fields[1], fields[3], kind=fields[2]) + if fields[2] == ' base class ': + H.add_edge(fields[1], fields[3], kind=fields[2]) f.close() -import fileinput -for line in fileinput.input(files =('function-statics-db.txt','function-calls-db.txt')): - if not bfunc.search(line) : continue - fields = line.split("'") - if skip.search(fields[1]) or skip.search(fields[3]) : continue - if fields[2] == ' calls function ' : - G.add_edge(fields[1],fields[3],kind=' calls function ') - if getfunc.search(fields[3]) : - dataclassfuncs.add(fields[3]) - m = getfunc.match(fields[3]) - n = handle.match(m.group(1)) - if n : o = n.group(3) - else : o = m.group(1) - p = re.sub("class ","",o) - dataclass = re.sub("struct ","",p) - dataclasses.add(dataclass) - if fields[2] == ' overrides function ' : - if baseclass.search(fields[3]) : - G.add_edge(fields[1],fields[3],kind=' overrides function ') - if topfunc.search(fields[3]) : toplevelfuncs.add(fields[1]) - else : G.add_edge(fields[3],fields[1], kind=' calls override function ') - if fields[2] == ' static variable ' : - G.add_edge(fields[1],fields[3],kind=' static variable ') - statics.add(fields[3]) +for line in fileinput.input(files=('function-statics-db.txt', 'function-calls-db.txt')): + if not bfunc.search(line): + continue + fields = line.split("'") + if skip.search(fields[1]) or skip.search(fields[3]): + continue + if fields[2] == ' calls function ': + G.add_edge(fields[1], fields[3], kind=' calls function ') + if getfunc.search(fields[3]): + dataclassfuncs.add(fields[3]) + m = getfunc.match(fields[3]) + n = handle.match(m.group(1)) + if n: + o = n.group(3) + else: + o = m.group(1) + p = re.sub("class ", "", o) + dataclass = re.sub("struct ", "", p) + dataclasses.add(dataclass) + if fields[2] == ' overrides function ': + if baseclass.search(fields[3]): + G.add_edge(fields[1], fields[3], kind=' overrides function ') + if topfunc.search(fields[3]): + toplevelfuncs.add(fields[1]) + else: + G.add_edge(fields[3], fields[1], kind=' calls override function ') + if fields[2] == ' static variable ': + G.add_edge(fields[1], fields[3], kind=' static variable ') + statics.add(fields[3]) fileinput.close() -for n,nbrdict in G.adjacency(): - for nbr,eattr in nbrdict.items(): - if n in badfuncs or nbr in badfuncs : - if 'kind' in eattr and eattr['kind'] == ' overrides function ' : - print("'"+n+"'"+eattr['kind']+"'"+nbr+"'") - virtfuncs.add(nbr) +for n, nbrdict in G.adjacency(): + for nbr, eattr in nbrdict.items(): + if n in badfuncs or nbr in badfuncs: + if 'kind' in eattr and eattr['kind'] == ' overrides function ': + print("'"+n+"'"+eattr['kind']+"'"+nbr+"'") + virtfuncs.add(nbr) print() -for n,nbrdict in H.adjacency(): - for nbr,eattr in nbrdict.items(): - if n in badclasses and 'kind' in eattr and eattr['kind'] == ' base class ' : - virtclasses.add(nbr) +for n, nbrdict in H.adjacency(): + for nbr, eattr in nbrdict.items(): + if n in badclasses and 'kind' in eattr and eattr['kind'] == ' base class ': + virtclasses.add(nbr) -for n,nbrdict in H.adjacency(): - for nbr,eattr in nbrdict.items(): - if nbr in dclasses and 'kind' in eattr and eattr['kind'] == ' base class ' : - dclasses.add(n) +for n, nbrdict in H.adjacency(): + for nbr, eattr in nbrdict.items(): + if nbr in dclasses and 'kind' in eattr and eattr['kind'] == ' base class ': + dclasses.add(n) print("flagged functions found by checker") -for dfunc in sorted(badfuncs) : - print(dfunc) +for dfunc in sorted(badfuncs): + print(dfunc) print() print("flagged classes found by checker ") -for dclass in sorted(badclasses) : - print(dclass) +for dclass in sorted(badclasses): + print(dclass) print() -print("flagged classes found by checker union get") -for dclass in sorted(dclasses.intersection(badclasses)) : - print(dclass) +print("flagged classes found by checker union get") +for dclass in sorted(dclasses.intersection(badclasses)): + print(dclass) print() print("classes inheriting from flagged classes") for dclass in sorted(virtclasses): - print(dclass) + print(dclass) print() print("functions overridden by flagged functions") for dfunc in sorted(virtfuncs): - print(dfunc) + print(dfunc) print() for badclass in sorted(badclasses): - print("Event setup data class '"+badclass+"' is flagged.") - flaggedclasses.add(badclass) + print("Event setup data class '"+badclass+"' is flagged.") + flaggedclasses.add(badclass) print() for virtclass in sorted(virtclasses): - print("Event setup data class '"+virtclass+"' is flagged because inheriting class is flagged") - flaggedclasses.add(virtclass) + print("Event setup data class '"+virtclass + + "' is flagged because inheriting class is flagged") + flaggedclasses.add(virtclass) print() for badclass in sorted(badclasses): - for dataclass in sorted(dataclasses): - if H.has_node(badclass) and H.has_node(dataclass): - if nx.has_path(H,dataclass, badclass) : - print("Event setup data class '"+dataclass+"' contains or inherits from flagged class '"+badclass+"'.") - flaggedclasses.add(dataclass) - + for dataclass in sorted(dataclasses): + if H.has_node(badclass) and H.has_node(dataclass): + if nx.has_path(H, dataclass, badclass): + print("Event setup data class '" + dataclass + + "' contains or inherits from flagged class '" + badclass + "'.") + flaggedclasses.add(dataclass) + print() for dataclassfunc in sorted(dataclassfuncs): - for tfunc in sorted(toplevelfuncs): - if G.has_node(tfunc) and G.has_node(dataclassfunc) and nx.has_path(G,tfunc,dataclassfunc): - m = getfunc.match(dataclassfunc) - n = handle.match(m.group(1)) - if n : o = n.group(3) - else : o = m.group(1) - p = re.sub("class ","",o) - q = re.sub("struct ","",p) - dataclass = re.sub("\<.*\> ","",q) - for flaggedclass in sorted(flaggedclasses): - exact= r"^" + re.escape(flaggedclass) + r"$" - exactmatch=re.match(exact,dataclass) - if exactmatch: - print("Flagged event setup data class '"+dataclass+"' is accessed in call stack '", end=' ') - path = nx.shortest_path(G,tfunc,dataclassfunc) - for p in path: - print(p+"; ", end=' ') - print("' ", end=' ') - for key in G[tfunc].keys() : - if 'kind' in G[tfunc][key] and G[tfunc][key]['kind'] == ' overrides function ' : - print("'"+tfunc+"'"+G[tfunc][key]['kind']+"'"+key+"'", end=' ') - print("") - print("In call stack '", end=' ') - path = nx.shortest_path(G,tfunc,dataclassfunc) - for p in path: - print(p+"; ", end=' ') - print("' flagged event setup data class '"+dataclass+"' is accessed. ", end=' ') - for key in G[tfunc].keys() : - if 'kind' in G[tfunc][key] and G[tfunc][key]['kind'] == ' overrides function ' : - print("'"+tfunc+"'"+G[tfunc][key]['kind']+"'"+key+"'", end=' ') - print("") - + for tfunc in sorted(toplevelfuncs): + if G.has_node(tfunc) and G.has_node(dataclassfunc) and nx.has_path(G, tfunc, dataclassfunc): + m = getfunc.match(dataclassfunc) + n = handle.match(m.group(1)) + if n: + o = n.group(3) + else: + o = m.group(1) + p = re.sub("class ", "", o) + q = re.sub("struct ", "", p) + dataclass = re.sub(r"\<.*\> ", "", q) + for flaggedclass in sorted(flaggedclasses): + exact = r"^" + re.escape(flaggedclass) + r"$" + exactmatch = re.match(exact, dataclass) + if exactmatch: + print("Flagged event setup data class '"+dataclass + + "' is accessed in call stack '", end=' ') + path = nx.shortest_path(G, tfunc, dataclassfunc) + for p in path: + print(p+"; ", end=' ') + print("' ", end=' ') + for key in G[tfunc].keys(): + if 'kind' in G[tfunc][key] and G[tfunc][key]['kind'] == ' overrides function ': + print("'"+tfunc+"'"+G[tfunc][key] + ['kind']+"'"+key+"'", end=' ') + print("") + print("In call stack '", end=' ') + path = nx.shortest_path(G, tfunc, dataclassfunc) + for p in path: + print(p+"; ", end=' ') + print("' flagged event setup data class '" + + dataclass+"' is accessed. ", end=' ') + for key in G[tfunc].keys(): + if 'kind' in G[tfunc][key] and G[tfunc][key]['kind'] == ' overrides function ': + print("'"+tfunc+"'"+G[tfunc][key] + ['kind']+"'"+key+"'", end=' ') + print("") diff --git a/Utilities/StaticAnalyzers/scripts/edm-global-class.py b/Utilities/StaticAnalyzers/scripts/edm-global-class.py index 943b65c3bf85d..e9937bcc8e5bf 100755 --- a/Utilities/StaticAnalyzers/scripts/edm-global-class.py +++ b/Utilities/StaticAnalyzers/scripts/edm-global-class.py @@ -1,17 +1,23 @@ #! /usr/bin/env python3 from __future__ import print_function +import fileinput +import networkx as nx import re datacl = re.compile("^class ") bfunc = re.compile("^function ") mbcl = re.compile("(base|data|flagged) class") -farg = re.compile("(.*)\(\w+\)") -nsep = re.compile("\:\:") -topfunc = re.compile("::(produce|analyze|filter|beginLuminosityBlock|beginRun|beginStream)\(") -baseclass = re.compile("edm::(one::|stream::|global::)?ED(Producer|Filter|Analyzer)(Base)?") +farg = re.compile(r"(.*)\(\w+\)") +nsep = re.compile(r"\:\:") +topfunc = re.compile( + r"::(produce|analyze|filter|beginLuminosityBlock|beginRun|beginStream)\(") +baseclass = re.compile( + r"edm::(one::|stream::|global::)?ED(Producer|Filter|Analyzer)(Base)?") globalclass = re.compile("^edm::global::ED(Producer|Filter|Analyzer)$") -getfunc = re.compile("edm::eventsetup::EventSetupRecord::get\<.*\>\((.*)&\) const") +getfunc = re.compile( + r"edm::eventsetup::EventSetupRecord::get\<.*\>\((.*)&\) const") handle = re.compile("(.*),?class edm::ES(.*)Handle<(.*)>") -skip = re.compile("edm::serviceregistry::ServicesManager::MakerHolder::add() const") +skip = re.compile( + r"edm::serviceregistry::ServicesManager::MakerHolder::add() const") rootclass = re.compile("T(H1|Tree|Enum|DataType|Class|Branch|Named|File)") stdcl = re.compile("^std::(.*)[^>]$") stdptr = re.compile("(.*)_ptr$") @@ -28,130 +34,137 @@ dataclasses = set() flaggedclasses = set() globalclasses = set() -import networkx as nx -G=nx.DiGraph() -H=nx.DiGraph() -I=nx.DiGraph() +Gdg = nx.DiGraph() +Hdg = nx.DiGraph() +Idg = nx.DiGraph() f = open('class-checker.txt') for line in f: - if mbcl.search(line): - fields = line.split("'") - classname = fields[1] - funcname = fields[3] - badclasses.add(classname) - badfuncs.add(funcname) + if mbcl.search(line): + fields = line.split("'") + classname = fields[1] + funcname = fields[3] + badclasses.add(classname) + badfuncs.add(funcname) f.close() f = open('const-checker.txt') for line in f: - if mbcl.search(line): - fields = line.split("'") - classname = fields[1] - badclasses.add(classname) + if mbcl.search(line): + fields = line.split("'") + classname = fields[1] + badclasses.add(classname) f.close() f = open('classes.txt.dumperall') -for line in f : - if mbcl.search(line) : - fields = line.split("'") - if fields[2] == ' member data class ': - H.add_edge(fields[1],fields[3],kind=fields[2]) - if fields[2] == ' templated member data class ': - H.add_edge(fields[1],fields[3],kind=fields[2]) - if fields[2] == ' base class ': - H.add_edge(fields[1],fields[3],kind=fields[2]) - I.add_edge(fields[3],fields[1],kind=' derived class') - if globalclass.match(fields[3]): - globalclasses.add(fields[1]) - print("class '"+fields[1]+"' base class '"+fields[3]+"'") +for line in f: + if mbcl.search(line): + fields = line.split("'") + if fields[2] == ' member data class ': + Hdg.add_edge(fields[1], fields[3], kind=fields[2]) + if fields[2] == ' templated member data class ': + Hdg.add_edge(fields[1], fields[3], kind=fields[2]) + if fields[2] == ' base class ': + Hdg.add_edge(fields[1], fields[3], kind=fields[2]) + Idg.add_edge(fields[3], fields[1], kind=' derived class') + if globalclass.match(fields[3]): + globalclasses.add(fields[1]) + print("class '"+fields[1]+"' base class '"+fields[3]+"'") f.close() -import fileinput -for line in fileinput.input(files =('function-statics-db.txt','function-calls-db.txt')): - if not bfunc.search(line) : continue - fields = line.split("'") - if skip.search(fields[1]) or skip.search(fields[3]) : continue - if fields[2] == ' calls function ' : - G.add_edge(fields[1],fields[3],kind=' calls function ') - if getfunc.search(fields[3]) : - dataclassfuncs.add(fields[3]) - m = getfunc.match(fields[3]) - n = handle.match(m.group(1)) - if n : o = n.group(3) - else : o = m.group(1) - p = re.sub("class ","",o) - dataclass = re.sub("struct ","",p) - dataclasses.add(dataclass) - if fields[2] == ' overrides function ' : - if baseclass.search(fields[3]) : - G.add_edge(fields[1],fields[3],kind=' overrides function ') - if topfunc.search(fields[3]) : toplevelfuncs.add(fields[1]) - else : G.add_edge(fields[3],fields[1], kind=' calls override function ') - if fields[2] == ' static variable ' : - G.add_edge(fields[1],fields[3],kind=' static variable ') - statics.add(fields[3]) +for line in fileinput.input(files=('function-statics-db.txt', 'function-calls-db.txt')): + if not bfunc.search(line): + continue + fields = line.split("'") + if skip.search(fields[1]) or skip.search(fields[3]): + continue + if fields[2] == ' calls function ': + Gdg.add_edge(fields[1], fields[3], kind=' calls function ') + if getfunc.search(fields[3]): + dataclassfuncs.add(fields[3]) + m = getfunc.match(fields[3]) + n = handle.match(m.group(1)) + if n: + o = n.group(3) + else: + o = m.group(1) + p = re.sub("class ", "", o) + dataclass = re.sub("struct ", "", p) + dataclasses.add(dataclass) + if fields[2] == ' overrides function ': + if baseclass.search(fields[3]): + Gdg.add_edge(fields[1], fields[3], kind=' overrides function ') + if topfunc.search(fields[3]): + toplevelfuncs.add(fields[1]) + else: + Gdg.add_edge(fields[3], fields[1], kind=' calls override function ') + if fields[2] == ' static variable ': + Gdg.add_edge(fields[1], fields[3], kind=' static variable ') + statics.add(fields[3]) fileinput.close() - -for n,nbrdict in G.adjacency(): - for nbr,eattr in nbrdict.items(): - if n in badfuncs or nbr in badfuncs : - if 'kind' in eattr and eattr['kind'] == ' overrides function ' : - print("'"+n+"'"+eattr['kind']+"'"+nbr+"'") - virtfuncs.add(nbr) +for n, nbrdict in Gdg.adjacency(): + for nbr, eattr in nbrdict.items(): + if n in badfuncs or nbr in badfuncs: + if 'kind' in eattr and eattr['kind'] == ' overrides function ': + print("'"+n+"'"+eattr['kind']+"'"+nbr+"'") + virtfuncs.add(nbr) print() print("-----------------------------------------------") print("flagged functions found by checker") print("-----------------------------------------------") -for dfunc in sorted(badfuncs) : - print(dfunc) +for dfunc in sorted(badfuncs): + print(dfunc) print() print("-----------------------------------------------") print("flagged classes found by checker ") print("-----------------------------------------------") -for dclass in sorted(badclasses) : - print(dclass) +for dclass in sorted(badclasses): + print(dclass) print() nodes = sorted(badclasses) for node in nodes: - visited = set() - if node in visited: - continue - visited.add(node) - if node in H : stack = [(node,iter(H[node]))] - if node in I : - Q=nx.dfs_preorder_nodes(I,node) - for q in Q: - if q in H : - stack.append( ( q, iter( H[q] ) ) ) - while stack: - parent,children = stack[-1] - try: - child = next(children) - if globalclass.search(child): visited.add(child) - if rootclass.search(child): visited.add(child) - if child not in visited: - visited.add(child) - stack.append( ( child, iter( H[child] ) ) ) - kind=H[parent][child]['kind'] - if stdptr.search(kind): - if child in I : - Q=nx.dfs_preorder_nodes(I,child) - for q in Q : - if q in H : - stack.append( ( q, iter( H[q] ) ) ) - except StopIteration: - stack.pop() - print("flagged class "+node+" contains or inherits from classes ", end=' ') - for v in visited : print(v+",", end=' ') - print("\n\n") - for v in sorted(visited) : - if v in globalclasses: - print("EDM global class '"+v+"' is flagged because it is connected to flagged class '"+node+"'") - + visited = set() + if node in visited: + continue + visited.add(node) + if node in Hdg: + stack = [(node, iter(Hdg[node]))] + if node in Idg: + Qdg = nx.dfs_preorder_nodes(Idg, node) + for q in Qdg: + if q in Hdg: + stack.append((q, iter(Hdg[q]))) + while stack: + parent, children = stack[-1] + try: + child = next(children) + if globalclass.search(child): + visited.add(child) + if rootclass.search(child): + visited.add(child) + if child not in visited: + visited.add(child) + stack.append((child, iter(Hdg[child]))) + kind = Hdg[parent][child]['kind'] + if stdptr.search(kind): + if child in Idg: + Qdg = nx.dfs_preorder_nodes(Idg, child) + for q in Qdg: + if q in Hdg: + stack.append((q, iter(Hdg[q]))) + except StopIteration: + stack.pop() + print("flagged class " + node + " contains or inherits from classes ", end=' ') + for v in visited: + print(v + ",", end=' ') + print("\n\n") + for v in sorted(visited): + if v in globalclasses: + print("EDM global class '" + v + + "' is flagged because it is connected to flagged class '" + node + "'") diff --git a/Utilities/StaticAnalyzers/scripts/postprocess-scan-build.py b/Utilities/StaticAnalyzers/scripts/postprocess-scan-build.py index 8e04f69b9d2ab..d1a4f9c53e4ed 100755 --- a/Utilities/StaticAnalyzers/scripts/postprocess-scan-build.py +++ b/Utilities/StaticAnalyzers/scripts/postprocess-scan-build.py @@ -1,48 +1,49 @@ #!/usr/bin/env python3 from bs4 import BeautifulSoup -import sys, os -url=os.path.abspath(sys.argv[1]) +import sys +import os +url = os.path.abspath(sys.argv[1]) report_dir = os.path.dirname(url) -page=open(url) -soup=BeautifulSoup(page.read(),'html.parser') +page = open(url) +soup = BeautifulSoup(page.read(), 'html.parser') page.close() -seen=dict() -tables=soup.find_all('table',recursive=True) +seen = dict() +tables = soup.find_all('table', recursive=True) -rowheader=tables[2].find('thead') -rowheaders=rowheader.find_all('tr') +rowheader = tables[2].find('thead') +rowheaders = rowheader.find_all('tr') htag = soup.new_tag('td') -htag.string='Num reports' -htag['class']='Q' -rowheaders[-1].insert(7,htag) -sortable=rowheaders[-1].find_all('span') +htag.string = 'Num reports' +htag['class'] = 'Q' +rowheaders[-1].insert(7, htag) +sortable = rowheaders[-1].find_all('span') sortable[0].string.replace_with(' ▾') rowsbody = tables[2].find('tbody') -rows=rowsbody.find_all('tr') +rows = rowsbody.find_all('tr') for row in rows: - cells=row.find_all('td') - key=str(cells[2])+str(cells[3])+str(cells[4]) + cells = row.find_all('td') + key = str(cells[2])+str(cells[3])+str(cells[4]) if key in seen.keys(): - seen[key]=seen[key]+1 - href = cells[6].find('a',href=True) + seen[key] = seen[key]+1 + href = cells[6].find('a', href=True) if href: - report = href['href'].split("#")[0] - report_file = os.path.join(report_dir, report) - if report.startswith("report-") and os.path.exists(report_file): - os.remove(report_file) + report = href['href'].split("#")[0] + report_file = os.path.join(report_dir, report) + if report.startswith("report-") and os.path.exists(report_file): + os.remove(report_file) row.decompose() else: - seen[key]=1 + seen[key] = 1 rowsbody = tables[2].find('tbody') -rows=rowsbody.find_all('tr') +rows = rowsbody.find_all('tr') for row in rows: - cells=row.find_all('td') - key=str(cells[2])+str(cells[3])+str(cells[4]) + cells = row.find_all('td') + key = str(cells[2])+str(cells[3])+str(cells[4]) tag = soup.new_tag('td') - tag.string='{}'.format(seen[key]) - tag['class']='Q' - row.insert(3,tag) + tag.string = '{}'.format(seen[key]) + tag['class'] = 'Q' + row.insert(3, tag) print(soup.prettify(formatter=None)) diff --git a/Utilities/StaticAnalyzers/scripts/statics.py b/Utilities/StaticAnalyzers/scripts/statics.py index ccfef5ea50664..8acaff30d1d1e 100755 --- a/Utilities/StaticAnalyzers/scripts/statics.py +++ b/Utilities/StaticAnalyzers/scripts/statics.py @@ -1,63 +1,75 @@ #! /usr/bin/env python3 from __future__ import print_function +import fileinput +import networkx as nx from builtins import range import re -topfunc = re.compile("::(produce|analyze|filter|beginLuminosityBlock|beginRun|beginStream|streamBeginRun|streamBeginLuminosityBlock|streamEndRun|streamEndLuminosityBlock|globalBeginRun|globalEndRun|globalBeginLuminosityBlock|globalEndLuminosityBlock|endRun|endLuminosityBlock)\(") -baseclass = re.compile("edm::(one::|stream::|global::)?ED(Producer|Filter|Analyzer)(Base)?") -farg = re.compile("\(.*\)") +topfunc = re.compile(r"::(produce|analyze|filter|beginLuminosityBlock|beginRun|beginStream|streamBeginRun|streamBeginLuminosityBlock|streamEndRun|streamEndLuminosityBlock|globalBeginRun|globalEndRun|globalBeginLuminosityBlock|globalEndLuminosityBlock|endRun|endLuminosityBlock)\(") +baseclass = re.compile( + "edm::(one::|stream::|global::)?ED(Producer|Filter|Analyzer)(Base)?") +farg = re.compile(r"\(.*\)") fre = re.compile("function") statics = set() toplevelfuncs = set() -skipfunc = re.compile("(edm::(LuminosityBlock::|Run::|Event::|Principal::)getBy(Label|Token))|(fwlite::|edm::EDProductGetter::getIt|edm::Event::|edm::eventsetup::EventSetupRecord::get|edm::eventsetup::DataProxy::getImpl|edm::EventPrincipal::unscheduledFill|edm::ServiceRegistry::get|edm::eventsetup::EventSetupRecord::getImplementation|edm::eventsetup::EventSetupRecord::getFromProxy|edm::eventsetup::DataProxy::get|edm::serviceregistry::ServicesManager::MakerHolder::add|(cond::service::PoolDBOutputService::(writeOne|appendSinceTime|tagInfo))|edm::EventProcessor::|edm::SubProcess::)") -skipfuncs=set() +skipfunc = re.compile(r"(edm::(LuminosityBlock::|Run::|Event::|Principal::)getBy(Label|Token))|(fwlite::|edm::EDProductGetter::getIt|edm::Event::|edm::eventsetup::EventSetupRecord::get|edm::eventsetup::DataProxy::getImpl|edm::EventPrincipal::unscheduledFill|edm::ServiceRegistry::get|edm::eventsetup::EventSetupRecord::getImplementation|edm::eventsetup::EventSetupRecord::getFromProxy|edm::eventsetup::DataProxy::get|edm::serviceregistry::ServicesManager::MakerHolder::add|(cond::service::PoolDBOutputService::(writeOne|appendSinceTime|tagInfo))|edm::EventProcessor::|edm::SubProcess::)") +skipfuncs = set() -import networkx as nx -G=nx.DiGraph() +G = nx.DiGraph() -import fileinput -for line in fileinput.input(files =('function-statics-db.txt','function-calls-db.txt')): - if fre.search(line): - fields = line.split("'") - if topfunc.search(fields[1]) and not baseclass.search(fields[1]): toplevelfuncs.add(fields[1]) - if fields[2] == ' calls function ': - if skipfunc.search(line) : skipfuncs.add(line) - else : G.add_edge(fields[1],fields[3],kind=fields[2]) - if fields[2] == ' overrides function ' : - if baseclass.search(fields[3]) : - if topfunc.search(fields[3]) : toplevelfuncs.add(fields[1]) - G.add_edge(fields[1],fields[3],kind=fields[2]) - else : - if skipfunc.search(line) : skipfuncs.add(line) - else : G.add_edge(fields[3],fields[1],kind=' calls function ') - if fields[2] == ' static variable ' : - G.add_edge(fields[1],fields[3],kind=fields[2]) - statics.add(fields[3]) - if fields[2] == ' known thread unsafe function ' : - G.add_edge(fields[1],fields[3],kind=' known thread unsafe function ') - statics.add(fields[3]) +for line in fileinput.input(files=('function-statics-db.txt', 'function-calls-db.txt')): + if fre.search(line): + fields = line.split("'") + if topfunc.search(fields[1]) and not baseclass.search(fields[1]): + toplevelfuncs.add(fields[1]) + if fields[2] == ' calls function ': + if skipfunc.search(line): + skipfuncs.add(line) + else: + G.add_edge(fields[1], fields[3], kind=fields[2]) + if fields[2] == ' overrides function ': + if baseclass.search(fields[3]): + if topfunc.search(fields[3]): + toplevelfuncs.add(fields[1]) + G.add_edge(fields[1], fields[3], kind=fields[2]) + else: + if skipfunc.search(line): + skipfuncs.add(line) + else: + G.add_edge(fields[3], fields[1], kind=' calls function ') + if fields[2] == ' static variable ': + G.add_edge(fields[1], fields[3], kind=fields[2]) + statics.add(fields[3]) + if fields[2] == ' known thread unsafe function ': + G.add_edge(fields[1], fields[3], + kind=' known thread unsafe function ') + statics.add(fields[3]) fileinput.close() for tfunc in sorted(toplevelfuncs): - for static in sorted(statics): - if G.has_node(tfunc) and G.has_node(static) and nx.has_path(G,tfunc,static): - path = nx.shortest_path(G,tfunc,static) - - print("Non-const static variable \'"+re.sub(farg,"()",static)+"' is accessed in call stack '", end=' ') - for i in range(0,len(path)-1) : - print(re.sub(farg,"()",path[i])+G[path[i]][path[i+1]]['kind'], end=' ') - print(re.sub(farg,"()",path[i+1])+"' ,", end=' ') - for key in G[tfunc].keys() : - if 'kind' in G[tfunc][key] and G[tfunc][key]['kind'] == ' overrides function ' : - print("'"+re.sub(farg,"()",tfunc)+"' overrides '"+re.sub(farg,"()",key)+"'", end=' ') - print() + for static in sorted(statics): + if G.has_node(tfunc) and G.has_node(static) and nx.has_path(G, tfunc, static): + path = nx.shortest_path(G, tfunc, static) - print("In call stack ' ", end=' ') - for i in range(0,len(path)-1) : - print(re.sub(farg,"()",path[i])+G[path[i]][path[i+1]]['kind'], end=' ') - print(re.sub(farg,"()",path[i+1])+"' is accessed ,", end=' ') - for key in G[tfunc].keys() : - if 'kind' in G[tfunc][key] and G[tfunc][key]['kind'] == ' overrides function ' : - print("'"+re.sub(farg,"()",tfunc)+"' overrides '"+re.sub(farg,"()",key)+"'", end=' ') - print() + print("Non-const static variable \'"+re.sub(farg, "()", + static)+"' is accessed in call stack '", end=' ') + for i in range(0, len(path)-1): + print(re.sub(farg, "()", path[i]) + + G[path[i]][path[i+1]]['kind'], end=' ') + print(re.sub(farg, "()", path[i+1])+"' ,", end=' ') + for key in G[tfunc].keys(): + if 'kind' in G[tfunc][key] and G[tfunc][key]['kind'] == ' overrides function ': + print("'"+re.sub(farg, "()", tfunc)+"' overrides '" + + re.sub(farg, "()", key)+"'", end=' ') + print() + print("In call stack ' ", end=' ') + for i in range(0, len(path)-1): + print(re.sub(farg, "()", path[i]) + + G[path[i]][path[i+1]]['kind'], end=' ') + print(re.sub(farg, "()", path[i+1])+"' is accessed ,", end=' ') + for key in G[tfunc].keys(): + if 'kind' in G[tfunc][key] and G[tfunc][key]['kind'] == ' overrides function ': + print("'"+re.sub(farg, "()", tfunc)+"' overrides '" + + re.sub(farg, "()", key)+"'", end=' ') + print() diff --git a/Utilities/StaticAnalyzers/scripts/symbols.py b/Utilities/StaticAnalyzers/scripts/symbols.py index bd186c43f8909..f1d301c28afd5 100755 --- a/Utilities/StaticAnalyzers/scripts/symbols.py +++ b/Utilities/StaticAnalyzers/scripts/symbols.py @@ -1,6 +1,7 @@ #! /usr/bin/env python3 from __future__ import print_function +import networkx as nx import collections import os import re @@ -11,7 +12,7 @@ symbol_re = r"(?P[a-zA-Z0-9_.$@]+)" symbol_demunged_re = r"(?P[a-zA-Z0-9_.$@:&()<>{}\[\]|^!%,~*+-=# ]+)" symbols_re_skip = re.compile("(@@)") -nm_line_re = re.compile(r"\s+".join([addr_re, code_re, symbol_re]) + "\s*$", +nm_line_re = re.compile(r"\s+".join([addr_re, code_re, symbol_re]) + r"\s*$", re.I) ldd_line_re = re.compile(r"\s+(.*) => (.*) \(0x") @@ -20,76 +21,86 @@ dependencies = collections.defaultdict(set) libraries = collections.defaultdict(set) + def get_symbols(fname): lines = subprocess.check_output(["nm", "-g", fname]) - for l in lines.splitlines(): - m = nm_line_re.match(l) - if not m : continue + for line in lines.splitlines(): + m = nm_line_re.match(line) + if not m: + continue symbol = m.group('symbol') if m.group('code') == 'U': requires[os.path.basename(fname)].add(symbol) else: provides[symbol].add(os.path.basename(fname)) + def get_libraries(fname): - lines = subprocess.check_output(["ldd",fname]) - for l in lines.splitlines(): - m = ldd_line_re.match(l) - if not m: continue + lines = subprocess.check_output(["ldd", fname]) + for line in lines.splitlines(): + m = ldd_line_re.match(line) + if not m: + continue library = m.group(2) - libraries[os.path.basename(fname)].add(os.path.basename(library.rstrip('\r\n'))) + libraries[os.path.basename(fname)].add( + os.path.basename(library.rstrip('\r\n'))) -paths=os.environ['LD_LIBRARY_PATH'].split(':') +paths = os.environ['LD_LIBRARY_PATH'].split(':') for p in paths: for dirpath, dirnames, filenames in os.walk(p): for f in filenames: - fpth=os.path.realpath(os.path.join(dirpath,f)) + fpth = os.path.realpath(os.path.join(dirpath, f)) filetype = subprocess.check_output(["file", fpth]) - if filetype.find("dynamically linked") >= 0 : + if filetype.find("dynamically linked") >= 0: get_symbols(fpth) get_libraries(fpth) for fname, symbols in requires.items(): - deps=set() + deps = set() for library in libraries[fname]: - for s in symbols : - if library in provides[s] : deps.add(library) - dependencies[fname]=deps - print(fname + ' : primary dependencies : ' + ', '.join(sorted(dependencies[fname]))+'\n') + for s in symbols: + if library in provides[s]: + deps.add(library) + dependencies[fname] = deps + print(fname + ' : primary dependencies : ' + + ', '.join(sorted(dependencies[fname]))+'\n') unmet = set() demangled = set() - for s in symbols: - if s not in provides and not symbols_re_skip.search(s) : unmet.add(s) + for s in symbols: + if s not in provides and not symbols_re_skip.search(s): + unmet.add(s) for u in sorted(unmet): - dm = subprocess.check_output(["c++filt",u]) + dm = subprocess.check_output(["c++filt", u]) demangled.add(dm.rstrip('\r\n')) - if demangled : print(fname + ': undefined : ' + ', '.join(sorted(demangled))) + if demangled: + print(fname + ': undefined : ' + ', '.join(sorted(demangled))) -import networkx as nx -G=nx.DiGraph() -for key,values in dependencies.items(): +G = nx.DiGraph() +for key, values in dependencies.items(): G.add_node(key) - for val in values: G.add_edge(key,val) + for val in values: + G.add_edge(key, val) for node in nx.nodes_iter(G): - s = nx.dfs_successors(G,node) - deps=set() - if s : - for key,vals in s.items() : - if key != node : deps.add(key) - for v in vals : + s = nx.dfs_successors(G, node) + deps = set() + if s: + for key, vals in s.items(): + if key != node: + deps.add(key) + for v in vals: deps.add(v) print(node + ': primary and secondary dependencies :' + ', '.join(sorted(deps))) -import pydot -H=nx.DiGraph() -for key,values in dependencies.items(): +H = nx.DiGraph() +for key, values in dependencies.items(): H.add_node(os.path.basename(key)) - for val in values: H.add_edge(os.path.basename(key),os.path.basename(val)) + for val in values: + H.add_edge(os.path.basename(key), os.path.basename(val)) for node in nx.nodes_iter(H): - T = nx.dfs_tree(H,node) + T = nx.dfs_tree(H, node) name = node + ".dot" - nx.write_dot(T,name) + nx.write_dot(T, name)