1
1

callvalidator.py 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498
  1. #!/usr/bin/env python
  2. # usage:callvalidator.py source=<src_input_dir> [file=<filename> | list=<listfilename>] [folder=<folder>]
  3. # tries to determine whether calls like gt 'location', 'event' are valid or not.
  4. from os import listdir
  5. from os.path import join
  6. import sys
  7. import re
  8. from io import open
  9. from time import perf_counter_ns
  10. import xml.etree.ElementTree as ET
  11. from textwrap import wrap
  12. from typing import List
  13. from threading import Thread
  14. from difflib import SequenceMatcher
  15. def createcallFileList(callFileList: List) -> None:
  16. validationTarget = ''
  17. if len(sys.argv) > 2 and "file=" in str(sys.argv[2]):
  18. validationTarget = sys.argv[2].replace("file=", "")
  19. callFileList = [join(vdir, sys.argv[2].replace("file=", ""))]
  20. validate = 'file'
  21. if len(sys.argv) > 2 and "list=" in str(sys.argv[2]):
  22. validationTarget = sys.argv[2].replace("list=", "")
  23. validationListFile = join(sys.argv[3].replace("folder=", ""), sys.argv[2].replace("list=", ""))
  24. try:
  25. tree = ET.parse(validationListFile)
  26. root = tree.getroot()
  27. for file in root.iter('Location'):
  28. callFileList.append(join(file.attrib['folder'] if "folder" in file.attrib.keys() else idir, "%s.qsrc" % file.attrib['name'].replace('$', '_')))
  29. except:
  30. try:
  31. with open(validationListFile, 'r', encoding='utf-8') as ifile:
  32. lines = ifile.readlines()
  33. for index, line in enumerate(lines):
  34. temp = line.strip(" \r\n\t").split(";")
  35. file = temp[0].strip(" \t")
  36. if ".qsrc" not in file:
  37. raise SyntaxError()
  38. raise SyntaxError("Incorrect file content: '<filename>[; <folder>] expected, found %s." % line,(validationListFile, index, 1, 'if ".qsrc" not in temp[0].strip(" \t"):', index, len(line)))
  39. if len(temp) == 2:
  40. callFileList.append(join(temp[1].strip(" \t"),file))
  41. else:
  42. callFileList.append(join(idir,file))
  43. #endfor
  44. #endwith
  45. except SyntaxError as e:
  46. raise SystemExit("Invalid list filed: %s" % e.msg)
  47. #endtry
  48. #endtry
  49. validate = 'list'
  50. #endif <- "list="
  51. if validate == 'all':
  52. validationTarget = idir
  53. callFileList = [join(idir,f) for f in listdir(idir) if ".qsrc" in f]
  54. #endif
  55. #enddef createValidationList()
  56. def buildValidCallsList(locationFileList) -> None:
  57. # Build the list of valid calls
  58. linecount = []
  59. for file in locationFileList:
  60. with open(join(idir, file), 'rt', encoding='utf-8', newline="\n") as ifile:
  61. lines = ifile.readlines()
  62. linecount.append({"file": file, "lc": len(lines)})
  63. location = lines[0].strip(' #\r\n').strip(' ')
  64. if location not in locationList:
  65. locationList.append(location)
  66. lcLocationList.append(location.lower())
  67. blockComment = False
  68. for line in lines:
  69. #getting locations
  70. workline = line.strip(' \t\r\n')
  71. locationsInLine = []
  72. isCommentedLine = blockComment
  73. match = re.search(calledLinePattern, workline)
  74. if match != None:
  75. potLine = re.findall(findLocationValues, workline)
  76. if not blockComment and re.search(blockCommentStart, workline) != None:
  77. blockComment = True
  78. isCommentedLine = True
  79. if not blockComment and re.search(commentLine, workline) != None: isCommentedLine = True
  80. if potLine != None: locationsInLine = potLine
  81. if blockComment and re.search(blockCommentEnd, workline) != None: blockComment = False
  82. #endif
  83. for match in locationsInLine:
  84. record = {"location": location.lower(), "function": match, "isCommentLine": isCommentedLine}
  85. recordInv = {"location": location.lower(), "function": match, "isCommentLine": not isCommentedLine}
  86. if record not in validLocationCalls and recordInv not in validLocationCalls:
  87. validLocationCalls.append({"location": location,
  88. "function": match,
  89. "isCommentLine": isCommentedLine})
  90. lcValidLocationCalls.append({"location": location.lower(), "function": match.lower()})
  91. elif recordInv in validLocationCalls:
  92. # if commentLine of recordInv is TRUE then set it to false.
  93. # if commentLine of recordInv is FALSE then leave it alone
  94. if recordInv["isCommentLine"]:
  95. index = validLocationCalls.index(recordInv)
  96. validLocationCalls[index]['isCommentLine'] = False
  97. #endif
  98. #endfor <- match in locationsInLine
  99. #endfor <- lineNo, line
  100. #endwith
  101. #endfor
  102. #enddef buildValidCallsList()
  103. def validCallsByLocationTXT(validLocationCalls: List) -> None:
  104. with open('valid-calls-by-location.txt', 'w', encoding='utf-8') as ofile:
  105. currLoc = ''
  106. ofile.write("---- List of valid calls by location\n")
  107. for call in validLocationCalls:
  108. if call['location'] != currLoc:
  109. ofile.write("\n")
  110. ofile.write(" ---- %s -------------\n" % call['location'])
  111. currLoc = call['location']
  112. ofile.write(" '%s': Commented Line: %s\n" % (call['function'], call['isCommentLine']))
  113. #enddef validCallsByLocationTXT
  114. def validCallsByLocationMD(validLocationCalls: List) -> None:
  115. with open('valid-calls-by-location.md', 'w', encoding='utf-8') as ofile:
  116. ofile.write("## Valid Calls per Location")
  117. currLoc = ''
  118. for call in validLocationCalls:
  119. if call['location'] != currLoc:
  120. currLoc = call['location']
  121. ofile.write("\n")
  122. ofile.write("### %s" % currLoc)
  123. ofile.write("\n")
  124. ofile.write('| "Function" | Is Comment |\n')
  125. ofile.write("| ---------- | ---------- |\n")
  126. ofile.write("| `'%s'` | %s |\n" % (call['function'], call['isCommentLine']))
  127. ofile.write("\n")
  128. #enddef validCallsByLocationMD
  129. def validCallsForTest(validLocationCalls) -> None:
  130. with open('validcallsfortesting.txt', 'w', encoding='utf-8') as ofile:
  131. for call in validLocationCalls:
  132. if not call['isCommentLine']: ofile.write("%s:%s\n" % (call['location'], call['function']))
  133. #enddef validCallsForTest
  134. def invalidCallsMissingLocationsTXT(invalidLocations: List, callsWithTypos: List, missingLocations: List) -> None:
  135. with open('invalid-calls-missing-locations.txt', 'w', encoding='utf-8') as ofile:
  136. ofile.write("----- Summary ------------------------------------------\n")
  137. ofile.write(" Locations called incorrectly : {:>4}\n".format(len(invalidLocations)))
  138. ofile.write(" Location/File doesn't exist : {:>4} [{:>3.2f}%%]\n".format(invalidLocationMissing,(invalidLocationMissing/len(invalidLocations))*100))
  139. ofile.write(" Commented out code : {:>4} [{:>3.2f}%%]\n".format(invalidCommentLine, (invalidCommentLine/len(invalidLocations))*100))
  140. ofile.write(" Value not expected/handled : {:>4} [{:>3.2f}%%]\n".format(invalidFunctionMissing, (invalidFunctionMissing/len(invalidLocations))*100))
  141. ofile.write("--------------------------------------------------------\n")
  142. ofile.write(" Locations making incorrect calls : {:>4}\n".format(locationsMakingInvalidCalls))
  143. ofile.write(" Total number of invalid calls : {:>4}\n".format(len(invalidCallsMade)))
  144. ofile.write(" Calls made to non-existing location: {:>4} [{:>3.2f}%%]\n".format(invalidCallsToMissingLocation, (invalidCallsToMissingLocation/len(invalidCallsMade))*100))
  145. ofile.write(" Location or File name has typo : {:>4} [{:>3.2f}%%]\n".format(invalidCallsToDueToTypos,(invalidCallsToDueToTypos/len(invalidLocations))*100))
  146. ofile.write(" Calls made to commented location : {:>4} [{:>3.2f}%%]\n".format(invalidCallsToCommentedCode, (invalidCallsToCommentedCode/len(invalidCallsMade))*100))
  147. ofile.write(" Calls made with unexpected value : {:>4} [{:>3.2f}%%]\n".format(invalidCallsToMissingFunction, (invalidCallsToMissingFunction/len(invalidCallsMade))*100))
  148. ofile.write("\n")
  149. ofile.write(" Valid calls with typos : {:>4} [{:>3.2f}%%]\n".format(validCallsWithTypos, (validCallsWithTypos/(len(locationCallList)-len(invalidCallsMade)))*100))
  150. ofile.write("--------------------------------------------------------\n")
  151. ofile.write(" Missing locations called : {:>4}\n".format(len(missingLocations)))
  152. ofile.write("--------------------------------------------------------\n")
  153. ofile.write("\n")
  154. ofile.write("\n")
  155. ofile.write("{:^80}\n".format("---------- List of Invalid Calls by location ----------"))
  156. currLoc = ''
  157. for call in invalidLocations:
  158. if call['location'] != currLoc:
  159. ofile.write("\n")
  160. ofile.write(" ---- %s -------------\n" % call['location'])
  161. currLoc = call['location']
  162. wrapis = wrap(" {:<34}: {}\n".format("'%s'" % call['function'], call['reason']), 125, subsequent_indent="{:<31}".format(' '), break_long_words=True)
  163. for line in wrapis:
  164. ofile.write("%s\n" % line)
  165. #endfor <- call in invalidLocations
  166. ofile.write("\n")
  167. ofile.write("{:^125}\n".format("---------- Invalid Calls made by location ----------"))
  168. currLoc = ''
  169. for call in invalidCallsMade:
  170. if call['callinglocation'] != currLoc:
  171. ofile.write("\n")
  172. ofile.write(" ---- %s [%s]-------------\n" % (call['callinglocation'], call['file']))
  173. currLoc = call['callinglocation']
  174. ofile.write("Line {:>4}: {} '{}', '{}'\n".format(call['lineNo'], call['calltype'], call['location'], call['function']))
  175. for line in wrap("%s%s" % (call['reason'].replace("`", "'"), " %s" % call['typo_msg'] if call['func_typo'] else ''), 120, initial_indent=' ', subsequent_indent=' ', break_long_words=True):
  176. ofile.write("%s\n"%line)
  177. #endfor <- call in invalidCallsMade
  178. ofile.write("\n")
  179. ofile.write("{:^125}\n".format("---------- Calls with typos made by location ----------"))
  180. currLoc = ''
  181. for call in callsWithTypos:
  182. if call['callinglocation'] != currLoc:
  183. ofile.write("\n")
  184. ofile.write(" ---- %s [%s]-------------\n" % (call['callinglocation'], call['file']))
  185. currLoc = call['callinglocation']
  186. ofile.write("Line {:>4}: {}\n".format(call['lineNo'], call['typo_msg']))
  187. #endfor <- call in callsWithTypos
  188. ofile.write("\n")
  189. ofile.write("{:^80}\n".format("---------- List of Missing Locations ----------"))
  190. ofile.write("\n")
  191. for location in missingLocations:
  192. ofile.write(" {:<25} [{}] \n".format(location, join(idir, "%s.qsrc" % location)))
  193. #endwith
  194. #enddef invalidCallsMissingLocationsTXT
  195. def invalidCallsMissingLocationsMD(invalidLocations: List, callsWithTypos: List, missingLocations: List) -> None:
  196. with open('invalid-calls-missing-locations.md', 'w', encoding='utf-8') as ofile:
  197. ofile.write("## List of Invalid Calls\n")
  198. currLoc = ''
  199. for call in invalidLocations:
  200. if call['location'] != currLoc:
  201. currLoc = call['location']
  202. ofile.write("\n")
  203. ofile.write("### %s" % currLoc)
  204. ofile.write("\n")
  205. ofile.write('| "Function" | Is Comment |\n')
  206. ofile.write("| ---------- | ---------- |\n")
  207. ofile.write("| `%s` | %s |\n" % (call['function'], call['reason']))
  208. ofile.write("\n")
  209. ofile.write("---\n")
  210. ofile.write("\n")
  211. ofile.write("## Calls made by locations\n")
  212. currLoc = ''
  213. for call in invalidCallsMade:
  214. if call['callinglocation'] != currLoc:
  215. currLoc = call['callinglocation']
  216. ofile.write("\n")
  217. ofile.write("### %s\n" % call['callinglocation'])
  218. ofile.write("\n")
  219. ofile.write("| File | Line No. | Call | Reason |\n")
  220. ofile.write("| ---- | -------- | ---- | ------ |\n")
  221. ofile.write("| {} | {:>4} | `{} '{}', '{}'` | {} |\n".format(call['file'], call['lineNo'], call['calltype'], call['location'], call['function'], call['reason']))
  222. ofile.write("\n")
  223. ofile.write("---\n")
  224. ofile.write("\n")
  225. ofile.write("## List of Missing Locations\n")
  226. ofile.write("\n")
  227. ofile.write ("| Location | File |\n")
  228. ofile.write ("| -------- | ---- |\n")
  229. for location in missingLocations:
  230. ofile.write("| %s | %s |\n" % (location, join(idir, "%s.qsrc" % location)))
  231. #endwith
  232. #enddef invalidCallsMissingLocationsMD
  233. runtime = [0,0,0,0,0,0,0,0,0,0,0,0,0,0]
  234. runtime[0] = perf_counter_ns()
  235. idir = ''
  236. validLocationCalls = []
  237. lcValidLocationCalls = []
  238. locationList = []
  239. lcLocationList = []
  240. locationFileList = []
  241. missingLocations = []
  242. locationCallList = []
  243. callFileList = []
  244. calledLocationList = []
  245. validate = 'all'
  246. locationsToIgnore = ['boystat', 'exp_gain']
  247. fileToIgnore = ["%s.qsrc" % loc for loc in locationsToIgnore]
  248. fileToIgnore += ['booty_call_after.qsrc','booty_call_condoms.qsrc','booty_call_cowgirl.qsrc','booty_call_cum.qsrc','booty_call_doggy.qsrc','booty_call_favorite_part.qsrc','booty_call_leave.qsrc','booty_call_miss.qsrc','booty_call_morning.qsrc','booty_call_pillow_talk.qsrc','booty_call_pillow_talk2.qsrc','booty_call_reactions.qsrc','booty_call_sex.qsrc','booty_call_shower.qsrc','booty_call_sms.qsrc','booty_call_start.qsrc','booty_call_stats.qsrc','booty_call_talk.qsrc','booty_call_virgin.qsrc','booty_call_work_talk1.qsrc']
  249. commentLine = "^(\s*!+)"
  250. blockCommentStart = "(^\s*!+.*{)"
  251. blockCommentEnd = "(}[^{]*$)"
  252. blockComment = False
  253. callLinePattern = "(gt|gs|xgt|xgs)(?:\s*(?:'|\"))(\w+)(?:'|\")(?:(?:,\s*)(?:'|\")(\w+)(?:'|\"))?"
  254. findCallLine = "(?:^\s*!+.*{)|(?:^[^!]*)(gt|gs|xgt|xgs)\s*(?:'|\")(\w+)(?:'|\")(?:(?:,\s*)(?:'|\")(\w+)(?:'|\"))?|(?:}[^{]*$)"
  255. calledLinePattern = "(?:^\s*!+.*{)|(?:(?:if|elseif)?\s*\(?\$?(?:ARGS|args)\[(?:0|i)\])(?:\s*(?:=|!|<|>|>=|=>|<=|=<)\s*(?:'|\"))(\w+)(?:'|\")|(?:}[^{]*$)"
  256. findLocationValues = "(?:(?:if|elseif)?\s*\(?\$?(?:ARGS|args)\[(?:0|i)\])(?:\s*(?:=|!|<|>|>=|=>|<=|=<)\s*(?:'|\"))(\w+)(?:'|\")"
  257. runtime_text = ['Initialising "global" variables', #0
  258. "Initialise with Arguments", #1
  259. "Build call file list and valid calls list (%d files)", #2
  260. "Building call list [%d files]", #3
  261. "Sorting and ordering the lists for the files", #4
  262. "Creating error type list [invalidLocations]", #5
  263. "Creating error type list [callsWithTypos]", #6
  264. "Generating file output", #7
  265. "All is finished", #8
  266. "", #9
  267. "", #10
  268. "", #11
  269. "", #12
  270. ""] #13
  271. runtime[1] = perf_counter_ns()
  272. print("0. %s: %d : %d: %d" % (runtime_text[0], runtime[0], runtime[1], runtime[1]-runtime[0] ))
  273. assert (len(sys.argv) == 2 or len(sys.argv) == 3 or len(sys.argv) == 4), "usage:\ncallvalidator.py source=<src_input_dir> [file=<filename> | list=<listfilename>] [folder=<folder>]"
  274. idir = str(sys.argv[1].replace("source=", ""))
  275. if len(sys.argv) == 4:
  276. vdir = str(sys.argv[3].replace("folder=", ""))
  277. else:
  278. vdir = idir
  279. locationFileList = [f for f in listdir(idir) if ".qsrc" in f and f not in fileToIgnore]
  280. callFileListThread = Thread(target=createcallFileList(callFileList), args=(callFileList,))
  281. validCallsListThread = Thread(buildValidCallsList(locationFileList), args=(locationFileList))
  282. runtime[2] = perf_counter_ns()
  283. print("1. %s: %d : %d: %d" % (runtime_text[1], runtime[1], runtime[2], runtime[2]-runtime[1] ))
  284. callFileListThread.start()
  285. validCallsListThread.start()
  286. callFileListThread.join()
  287. validCallsListThread.join()
  288. runtime[3] = perf_counter_ns()
  289. print("2. %s: %d : %d: %d" % (runtime_text[2] % len(locationFileList), runtime[2], runtime[3], runtime[3]-runtime[2] ))
  290. # build a list of all the calls happening
  291. for file in callFileList:
  292. with open(file, 'rt', encoding='utf-8', newline="\n") as ifile:
  293. lines = ifile.readlines()
  294. location = lines[0].strip(' #\r\n').strip(' ')
  295. for lineNo, line in enumerate(lines):
  296. workline = line.strip(' \t\n\r')
  297. match = None
  298. msg = ''
  299. temp_match = re.search(findCallLine, workline)
  300. if temp_match != None and not [res for res in locationsToIgnore if res in workline.lower()]:
  301. potLine = re.search(callLinePattern, workline)
  302. if not blockComment and re.search(blockCommentStart, line) != None: blockComment = True
  303. if not blockComment and potLine != None: match = potLine
  304. if blockComment and re.search(blockCommentEnd, line) != None: blockComment = False
  305. #endif
  306. if match != None:
  307. valid = 0
  308. func_typo = 0
  309. loc_typo = 0
  310. typo_msg = ""
  311. loc_msg = ""
  312. func_msg = "."
  313. reason = ""
  314. loc = match.group(2)
  315. func = '' if match.group(3) == None else match.group(3)
  316. searchRecord = {"location": loc.lower(), "function": func.lower()}
  317. if searchRecord['location'] not in lcLocationList:
  318. valid = -1
  319. reason = "Location `%s` doesn't exist - No file containing this location was found in the specified folder." % (loc)
  320. if loc not in missingLocations: missingLocations.append(loc)
  321. else:
  322. if loc not in locationList:
  323. index = lcLocationList.index(searchRecord['location'])
  324. loc_typo = 1
  325. loc_msg = "The location name is '%s' not '%s'" % (locationList[index], loc)
  326. #endif
  327. if searchRecord['function'] == '' and searchRecord not in lcValidLocationCalls:
  328. validLocationCalls.append({"location": loc, "function": '', "isCommentLine": False })
  329. lcValidLocationCalls.append(searchRecord)
  330. elif searchRecord in lcValidLocationCalls:
  331. index = lcValidLocationCalls.index(searchRecord)
  332. validL = validLocationCalls[index]
  333. if validL['function'] == func:
  334. if not validL['isCommentLine']:
  335. valid = 1
  336. reason = ""
  337. else:
  338. valid = -2
  339. reason = "`%s, %s` exists but is either commented out or is in a comment block" % (loc, func)
  340. #endif
  341. else:
  342. valid = -3
  343. reason = "Location `%s` exists but couldn't find any entrypoint expecting `%s` as $ARGS[0]. Please confirm whether this is a bug or not." % (loc, func)
  344. func_typo = 1
  345. if validL['isCommentLine']:
  346. if loc_typo:
  347. func_msg = " and $AGRS[0] has a case typo, it is '%s' not '%s'. Also, the correct call is a comment line or in a comment block." % (validL['function'], func)
  348. else:
  349. func_msg = "$AGRS[0] has a case typo, it should be '%s' not '%s'. Also, the correct call is a comment line or in a comment block." % (validL['function'], func)
  350. else:
  351. if loc_typo:
  352. func_msg = " and $AGRS[0] has a case typo, it is '%s' not '%s'." % (validL['function'], func)
  353. else:
  354. func_msg = "$AGRS[0] has a case typo, it should be '%s' not '%s'." % (validL['function'], func)
  355. #endif
  356. #endif
  357. else:
  358. strictness = 0.75
  359. possible_funcs = []
  360. for f in (f['function'] for f in validLocationCalls if f['location'] == loc.lower() and
  361. f['function'] not in possible_funcs and
  362. f['function'] != func and
  363. ("%s_" % func) not in f['function'] and
  364. ("%s_" % f['function']) not in func):
  365. similarity = SequenceMatcher(None, func, f, strictness)
  366. if similarity.ratio() >= strictness:
  367. possible_funcs.append("'%s'" % f)
  368. valid = -3
  369. reason = "Location `%s` exists but couldn't find any entrypoint expecting `%s` as $ARGS[0]. Please confirm whether this is a bug or not." % (loc, func)
  370. if len(possible_funcs) > 0:
  371. reason += "\nThere may be a typo in the $ARGS[0] value passed. Some similar expected $ARGS[0] values are: %s" % ", ".join(possible_funcs)
  372. #endif
  373. #endif
  374. if loc_typo or func_typo: typo_msg = "The call has %s. %s%s [%s]" % ("a typo" if func_typo != loc_typo else "typos", loc_msg, func_msg, "The call will fail" if func_typo else "The call will work")
  375. locationCallList.append({"callinglocation": location,
  376. "file": ifile.name,
  377. "lineNo": lineNo+1,
  378. "calltype": match.group(1),
  379. "location": loc,
  380. "function": func,
  381. "valid": valid,
  382. "reason": reason,
  383. "loc_typo": loc_typo,
  384. "func_typo": func_typo,
  385. "typo_msg": typo_msg})
  386. i = 0
  387. #endif# <- Match != None
  388. #endfor <- line in lines
  389. #endwith
  390. #endfor
  391. runtime[4] = perf_counter_ns()
  392. print("3. %s: %d : %d: %d" % (runtime_text[3] % len(callFileList), runtime[3],runtime[4], runtime[4]-runtime[3] ))
  393. missingLocations = sorted(missingLocations)
  394. validLocationCalls = sorted(validLocationCalls, key=lambda k: (k['location'].lower(), k['function']))
  395. invalidCallsMade = [call for call in locationCallList if call['valid'] < 0]
  396. invalidCallsMade = sorted(invalidCallsMade, key = lambda k: (k['callinglocation'].lower(), k['lineNo']))
  397. invalidLocations = []
  398. callsWithTypos = []
  399. invalidCallsToMissingLocation = 0
  400. invalidCallsToCommentedCode = 0
  401. invalidCallsToMissingFunction = 0
  402. invalidCallsToDueToTypos = 0
  403. invalidLocationMissing = 0
  404. invalidCommentLine = 0
  405. invalidFunctionMissing = 0
  406. locationsMakingInvalidCalls = 0
  407. validCallsWithTypos = 0
  408. runtime[5] = perf_counter_ns()
  409. print("4. %s: %d : %d: %d" % (runtime_text[4], runtime[4], runtime[5], runtime[5]-runtime[4] ))
  410. currLoc = ''
  411. for call in invalidCallsMade:
  412. record = {"location": call['location'], "function": call['function'], "valid": call['valid'], "reason": call['reason'], "func_type": call['func_typo'], "loc_typo": call['loc_typo'], "typo_msg": call['typo_msg']}
  413. if call['callinglocation'] != '':
  414. locationsMakingInvalidCalls += 1
  415. currLoc = call['callinglocation']
  416. if call['valid'] == -1: invalidCallsToMissingLocation += 1
  417. if call['valid'] == -2: invalidCallsToCommentedCode += 1
  418. if call['valid'] == -3: invalidCallsToMissingFunction += 1
  419. if call['func_typo'] : invalidCallsToDueToTypos += 1
  420. if record not in invalidLocations:
  421. invalidLocations.append(record)
  422. if record['valid'] == -1: invalidLocationMissing += 1
  423. if record['valid'] == -2: invalidCommentLine += 1
  424. if record['valid'] == -3: invalidFunctionMissing += 1
  425. invalidLocations = sorted(invalidLocations, key=lambda k: (k['location'].lower(), k['function']))
  426. runtime[6] = perf_counter_ns()
  427. print("5. %s: %d : %d: %d" % (runtime_text[5], runtime[5], runtime[6], runtime[6]-runtime[5] ))
  428. ##endfor for call in locationCallList
  429. callsWithTypos = [call for call in locationCallList if call['loc_typo'] or call['func_typo']]
  430. validCallsWithTypos = len(callsWithTypos)
  431. callsWithTypos = sorted(callsWithTypos, key=lambda k: (k['callinglocation'].lower(), k['lineNo']))
  432. runtime[7] = perf_counter_ns()
  433. print("6. %s: %d : %d: %d" % (runtime_text[6], runtime[6], runtime[7], runtime[7]-runtime[6] ))
  434. fileThread1 = Thread(target=validCallsByLocationTXT(validLocationCalls), args=(validLocationCalls,))
  435. #fileThread2 = Thread(target=validCallsByLocationMD(validLocationCalls), args=(validLocationCalls,))
  436. fileThread3 = Thread(target=validCallsForTest(validLocationCalls), args=(validLocationCalls,))
  437. fileThread4 = Thread(target=invalidCallsMissingLocationsTXT(invalidLocations, callsWithTypos, missingLocations), args=(invalidLocations, callsWithTypos, missingLocations))
  438. #fileThread5 = Thread(target=invalidCallsMissingLocationsMD(invalidLocations, callsWithTypos, missingLocations), args=(invalidLocations, callsWithTypos, missingLocations))
  439. fileThread1.start()
  440. #fileThread2.start()
  441. fileThread3.start()
  442. fileThread4.start()
  443. #fileThread5.start()
  444. fileThread1.join()
  445. #fileThread2.join()
  446. fileThread3.join()
  447. fileThread4.join()
  448. #fileThread5.join()
  449. runtime[8] = perf_counter_ns()
  450. print("7. %s: %d : %d: %d" % (runtime_text[7], runtime[7], runtime[8], runtime[8]-runtime[7] ))
  451. print("8. %s: %d : %d: %d" % (runtime_text[8], runtime[0], runtime[8], runtime[8]-runtime[0] ))