callvalidator.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346
  1. #!/usr/bin/env python
  2. # usage:callvalidator.py source=<src_input_dir> [file=<filename> | list=<listfilename>] [folder=<folder>]
  3. # tries to determine whether calls like gt 'location', 'event' are valid or not.
  4. from os import listdir
  5. from os.path import join
  6. import sys
  7. import re
  8. from io import open
  9. from time import perf_counter_ns
  10. import xml.etree.ElementTree as ET
  11. from textwrap import wrap
  12. runtime = [0,0,0,0,0,0,0,0,0,0]
  13. runtime[0] = perf_counter_ns()
  14. idir = ''
  15. validLocationCalls = []
  16. locationFileList = []
  17. locationList = []
  18. missingLocations = []
  19. locationCallList = []
  20. callFileList = []
  21. calledLocationList = []
  22. validate = 'all'
  23. locationsToIgnore = ['boystat', 'exp_gain']
  24. fileToIgnore = ["%s.qsrc" % loc for loc in locationsToIgnore]
  25. fileToIgnore += ['booty_call_after.qsrc','booty_call_condoms.qsrc','booty_call_cowgirl.qsrc','booty_call_cum.qsrc','booty_call_doggy.qsrc','booty_call_favorite_part.qsrc','booty_call_leave.qsrc','booty_call_miss.qsrc','booty_call_morning.qsrc','booty_call_pillow_talk.qsrc','booty_call_pillow_talk2.qsrc','booty_call_reactions.qsrc','booty_call_sex.qsrc','booty_call_shower.qsrc','booty_call_sms.qsrc','booty_call_start.qsrc','booty_call_stats.qsrc','booty_call_talk.qsrc','booty_call_virgin.qsrc','booty_call_work_talk1.qsrc']
  26. commentLine = "^(\s*!+)"
  27. blockCommentStart = "(^\s*!+.*{)"
  28. blockCommentEnd = "(}[^{]*$)"
  29. blockComment = False
  30. callLinePattern = "(gt|gs|xgt|xgs)(?:\s*(?:'|\"))(\w+)(?:'|\")(?:(?:,\s*)(?:'|\")(\w+)(?:'|\"))?"
  31. findCallLine = "(?:^\s*!+.*{)|(?:^[^!]*)(gt|gs|xgt|xgs)\s*(?:'|\")(\w+)(?:'|\")(?:(?:,\s*)(?:'|\")(\w+)(?:'|\"))?|(?:}[^{]*$)"
  32. calledLinePattern = "(?:^\s*!+.*{)|(?:(?:if|elseif)?\s*\(?\$?(?:ARGS|args)\[(?:0|i)\])(?:\s*(?:=|!|<|>|>=|=>|<=|=<)\s*(?:'|\"))(\w+)(?:'|\")|(?:}[^{]*$)"
  33. findLocationValues = "(?:(?:if|elseif)?\s*\(?\$?(?:ARGS|args)\[(?:0|i)\])(?:\s*(?:=|!|<|>|>=|=>|<=|=<)\s*(?:'|\"))(\w+)(?:'|\")"
  34. runtime_text = ["Setup started","Started callfile build","","","Sorting and ordering the lists for the files","","All is finished"]
  35. assert (len(sys.argv) == 2 or len(sys.argv) == 3 or len(sys.argv) == 4), "usage:\ncallvalidator.py source=<src_input_dir> [file=<filename> | list=<listfilename>] [folder=<folder>]"
  36. idir = str(sys.argv[1].replace("source=", ""))
  37. if len(sys.argv) == 4:
  38. vdir = str(sys.argv[3].replace("folder=", ""))
  39. else:
  40. vdir = idir
  41. validationTarget = ''
  42. runtime[1] = perf_counter_ns()
  43. print("%s: %d: %d" % (runtime_text[0], runtime[1], runtime[1]-runtime[0] ))
  44. if len(sys.argv) > 2 and "file=" in str(sys.argv[2]):
  45. validationTarget = sys.argv[2].replace("file=", "")
  46. callFileList = [join(vdir, sys.argv[2].replace("file=", ""))]
  47. validate = 'file'
  48. if len(sys.argv) > 2 and "list=" in str(sys.argv[2]):
  49. validationTarget = sys.argv[2].replace("list=", "")
  50. validationListFile = join(sys.argv[3].replace("folder=", ""), sys.argv[2].replace("list=", ""))
  51. try:
  52. tree = ET.parse(validationListFile)
  53. root = tree.getroot()
  54. for file in root.iter('Location'):
  55. callFileList.append(join(file.attrib['folder'] if "folder" in file.attrib.keys() else idir, "%s.qsrc" % file.attrib['name'].replace('$', '_')))
  56. except:
  57. try:
  58. with open(validationListFile, 'r', encoding='utf-8') as ifile:
  59. lines = ifile.readlines()
  60. for index, line in enumerate(lines):
  61. temp = line.strip(" \r\n\t").split(";")
  62. file = temp[0].strip(" \t")
  63. if ".qsrc" not in file:
  64. raise SyntaxError()
  65. raise SyntaxError("Incorrect file content: '<filename>[; <folder>] expected, found %s." % line,(validationListFile, index, 1, 'if ".qsrc" not in temp[0].strip(" \t"):', index, len(line)))
  66. if len(temp) == 2:
  67. callFileList.append(join(temp[1].strip(" \t"),file))
  68. else:
  69. callFileList.append(join(idir,file))
  70. #endfor
  71. #endwith
  72. except SyntaxError as e:
  73. raise SystemExit("Invalid list filed: %s" % e.msg)
  74. #endtry
  75. #endtry
  76. validate = 'list'
  77. #endif <- "list="
  78. if validate == 'all':
  79. validationTarget = idir
  80. callFileList = [join(idir,f) for f in listdir(idir) if ".qsrc" in f]
  81. #endif
  82. # Build the list of valid calls
  83. locationFileList = [f for f in listdir(idir) if ".qsrc" in f and f not in fileToIgnore]
  84. runtime[2] = perf_counter_ns()
  85. runtime_text[2] = "Building valid calls list [%d files]" % len(locationFileList)
  86. print("%s: %d: %d" % (runtime_text[1], runtime[2], runtime[2]-runtime[1] ))
  87. for file in locationFileList:
  88. with open(join(idir, file), 'rt', encoding='utf-8', newline="\n") as ifile:
  89. lines = ifile.readlines()
  90. location = lines[0].strip(' #\r\n').strip(' ')
  91. if location not in locationList: locationList.append(location)
  92. blockComment = False
  93. for lineNo, line in enumerate(lines):
  94. #getting locations
  95. workline = line.strip(' \t\r\n')
  96. locationsInLine = []
  97. isCommentedLine = blockComment
  98. match = re.search(calledLinePattern, workline)
  99. if match != None:
  100. potLine = re.findall(findLocationValues, workline)
  101. if not blockComment and re.search(blockCommentStart, workline) != None:
  102. blockComment = True
  103. isCommentedLine = True
  104. if not blockComment and re.search(commentLine, workline) != None: isCommentedLine = True
  105. if potLine != None: locationsInLine = potLine
  106. if blockComment and re.search(blockCommentEnd, workline) != None: blockComment = False
  107. #endif
  108. for match in locationsInLine:
  109. record = {"location": location, "function": match, "isCommentLine": isCommentedLine}
  110. recordInv = {"location": location, "function": match, "isCommentLine": not commentLine}
  111. if record not in validLocationCalls and recordInv not in validLocationCalls:
  112. validLocationCalls.append({"location": location,
  113. "function": match,
  114. "isCommentLine": isCommentedLine})
  115. elif recordInv in validLocationCalls:
  116. # if commentLine of recordInv is TRUE then set it to false.
  117. # if commentLine of recordInv is FALSE then leave it alone
  118. if recordInv["isCommentLine"]:
  119. index = validLocationCalls.index(recordInv)
  120. validLocationCalls[index]['isCommentLine'] = False
  121. #endif
  122. #endfor <- match in locationsInLine
  123. #endfor <- lineNo, line
  124. #endwith
  125. #endfor
  126. # build a list of all the calls happening
  127. runtime[3] = perf_counter_ns()
  128. runtime_text[3] = "Building call list [%d files]" % len(callFileList)
  129. print("%s: %d: %d" % (runtime_text[2], runtime[3], runtime[3]-runtime[2] ))
  130. for file in callFileList:
  131. with open(file, 'rt', encoding='utf-8', newline="\n") as ifile:
  132. lines = ifile.readlines()
  133. location = lines[0].strip(' #\r\n').strip(' ')
  134. for lineNo, line in enumerate(lines):
  135. workline = line.strip(' \t\n\r')
  136. match = None
  137. temp_match = re.search(findCallLine, workline)
  138. if temp_match != None and not [res for res in locationsToIgnore if res in workline.lower()]:
  139. potLine = re.search(callLinePattern, workline)
  140. if not blockComment and re.search(blockCommentStart, line) != None: blockComment = True
  141. if not blockComment and potLine != None: match = potLine
  142. if blockComment and re.search(blockCommentEnd, line) != None: blockComment = False
  143. #endif
  144. if match != None:
  145. valid = 0
  146. reason = ""
  147. searchRecordF = {"location": match.group(2), "function": '' if match.group(3) == None else match.group(3), "isCommentLine": False}
  148. searchRecordT = {"location": match.group(2), "function": '' if match.group(3) == None else match.group(3), "isCommentLine": True}
  149. if searchRecordF['function'] == '':
  150. if searchRecordF not in validLocationCalls: validLocationCalls.append(searchRecordF)
  151. valid = 1
  152. reason = ""
  153. elif searchRecordF['location'] not in locationList:
  154. valid = -1
  155. reason = "Location `%s` doesn't exist - No file containing this location was found in the specified folder." % (searchRecordF['location'])
  156. if searchRecordF['location'] not in missingLocations: missingLocations.append(searchRecordF['location'])
  157. elif searchRecordT in validLocationCalls:
  158. valid = -2
  159. reason = "`%s, %s` exists but is either commented out or is in a comment block" % (searchRecordT['location'], searchRecordT['function'])
  160. elif searchRecordF in validLocationCalls:
  161. valid = 1
  162. reason = ""
  163. else:
  164. valid = -3
  165. reason = "Location `%s` exists but couldn't find any entrypoint expecting `%s` as $ARGS[0]. Please confirm whether this is a bug or not." % (searchRecordF['location'], searchRecordF['function'])
  166. #endif
  167. locationCallList.append({"callinglocation": location,
  168. "file": ifile.name,
  169. "lineNo": lineNo+1,
  170. "calltype": match.group(1),
  171. "location": match.group(2),
  172. "function": '' if match.group(3) == None else match.group(3),
  173. "valid": valid,
  174. "reason": reason})
  175. #endif# <- Match != None
  176. #endfor <- line in lines
  177. #endwith
  178. #endfor
  179. runtime[4] = perf_counter_ns()
  180. print("%s: %d: %d" % (runtime_text[3], runtime[4], runtime[4]-runtime[3] ))
  181. missingLocations = sorted(missingLocations)
  182. validLocationCalls = sorted(validLocationCalls, key=lambda k: (k['location'].lower(), k['function']))
  183. invalidCallsMade = [call for call in locationCallList if call['valid'] < 0]
  184. invalidCallsMade = sorted(invalidCallsMade, key = lambda k: (k['callinglocation'].lower(), k['lineNo']))
  185. invalidLocations = []
  186. invalidCallsToMissingLocation = 0
  187. invalidCallsToCommentedCode = 0
  188. invalidCallsToMissingFunction = 0
  189. invalidLocationMissing = 0
  190. invalidCommentLine = 0
  191. invalidFunctionMissing = 0
  192. locationsMakingInvalidCalls = 0
  193. currLoc = ''
  194. for call in invalidCallsMade:
  195. record = {"location": call['location'], "function": call['function'], "valid": call['valid'], "reason": call['reason']}
  196. if call['callinglocation'] != '':
  197. locationsMakingInvalidCalls += 1
  198. currLoc = call['callinglocation']
  199. if call['valid'] == -1: invalidCallsToMissingLocation += 1
  200. if call['valid'] == -2: invalidCallsToCommentedCode += 1
  201. if call['valid'] == -3: invalidCallsToMissingFunction += 1
  202. if record not in invalidLocations:
  203. invalidLocations.append(record)
  204. if record['valid'] == -1: invalidLocationMissing += 1
  205. if record['valid'] == -2: invalidCommentLine += 1
  206. if record['valid'] == -3: invalidFunctionMissing += 1
  207. invalidLocations = sorted(invalidLocations, key=lambda k: (k['location'].lower(), k['function']))
  208. runtime[5] = perf_counter_ns()
  209. print("%s: %d: %d" % (runtime_text[4], runtime[5], runtime[5]-runtime[4] ))
  210. with open('valid-calls-by-location.txt', 'w', encoding='utf-8') as ofile:
  211. currLoc = ''
  212. ofile.write("---- List of valid calls by location\n")
  213. for call in validLocationCalls:
  214. if call['location'] != currLoc:
  215. ofile.write("\n")
  216. ofile.write(" ---- %s -------------\n" % call['location'])
  217. currLoc = call['location']
  218. ofile.write(" '%s': Commented Line: %s\n" % (call['function'], call['isCommentLine']))
  219. with open('valid-calls-by-location.md', 'w', encoding='utf-8') as ofile:
  220. ofile.write("## Valid Calls per Location")
  221. currLoc = ''
  222. for call in validLocationCalls:
  223. if call['location'] != currLoc:
  224. currLoc = call['location']
  225. ofile.write("\n")
  226. ofile.write("### %s" % currLoc)
  227. ofile.write("\n")
  228. ofile.write('| "Function" | Is Comment |\n')
  229. ofile.write("| ---------- | ---------- |\n")
  230. ofile.write("| `'%s'` | %s |\n" % (call['function'], call['isCommentLine']))
  231. ofile.write("\n")
  232. with open('invalid-calls-missing-locations.txt', 'w', encoding='utf-8') as ofile:
  233. ofile.write("----- Summary ------------------------------------------\n")
  234. ofile.write(" Locations called incorrectly : {:>3}\n".format(len(invalidLocations)))
  235. ofile.write(" Location/File doesn't exist : {:>3} [{:>3.2f}%%]\n".format(invalidLocationMissing,(invalidLocationMissing/len(invalidLocations))*100))
  236. ofile.write(" Commented out code : {:>3} [{:>3.2f}%%]\n".format(invalidCommentLine, (invalidCommentLine/len(invalidLocations))*100))
  237. ofile.write(" Value not expected/handled : {:>3} [{:>3.2f}%%]\n".format(invalidFunctionMissing, (invalidFunctionMissing/len(invalidLocations))*100))
  238. ofile.write("--------------------------------------------------------\n")
  239. ofile.write(" Locations making incorrect calls : {:>3}\n".format(locationsMakingInvalidCalls))
  240. ofile.write(" Total number of invalid calls : {:>3}\n".format(len(invalidCallsMade)))
  241. ofile.write(" Calls made to non-existing location: {:>3} [{:>3.2f}%%]\n".format(invalidCallsToMissingLocation, (invalidCallsToMissingLocation/len(invalidCallsMade))*100))
  242. ofile.write(" Calls made to commented location : {:>3} [{:>3.2f}%%]\n".format(invalidCallsToCommentedCode, (invalidCallsToCommentedCode/len(invalidCallsMade))*100))
  243. ofile.write(" Calls made with unexpected value : {:>3} [{:>3.2f}%%]\n".format(invalidCallsToMissingFunction, (invalidCallsToMissingFunction/len(invalidCallsMade))*100))
  244. ofile.write("--------------------------------------------------------\n")
  245. ofile.write(" Missing locations called : {:>3}\n".format(len(missingLocations)))
  246. ofile.write("--------------------------------------------------------\n")
  247. ofile.write("\n")
  248. ofile.write("\n")
  249. ofile.write("{:^80}\n".format("---------- List of Invalid Calls by location ----------"))
  250. currLoc = ''
  251. for call in invalidLocations:
  252. if call['location'] != currLoc:
  253. ofile.write("\n")
  254. ofile.write(" ---- %s -------------\n" % call['location'])
  255. currLoc = call['location']
  256. wrapis = wrap(" {:<34}: {}\n".format("'%s'" % call['function'], call['reason']), 125, subsequent_indent="{:<31}".format(' '), break_long_words=True)
  257. for line in wrapis:
  258. ofile.write("%s\n" % line)
  259. #endfor <- call in invalidLocations
  260. ofile.write("\n")
  261. ofile.write("{:^125}\n".format("---------- Calls made by location ----------"))
  262. currLoc = ''
  263. for call in invalidCallsMade:
  264. if call['callinglocation'] != currLoc:
  265. ofile.write("\n")
  266. ofile.write(" ---- %s [%s]-------------\n" % (call['callinglocation'], call['file']))
  267. currLoc = call['callinglocation']
  268. ofile.write("Line {:>4}: {} '{}', '{}'\n".format(call['lineNo'], call['calltype'], call['location'], call['function']))
  269. for line in wrap(call['reason'].replace("`", "'"), 90, initial_indent=' ', subsequent_indent=' ', break_long_words=True):
  270. ofile.write("%s\n"%line)
  271. #endfor <- call in invalidCallsMade
  272. ofile.write("\n")
  273. ofile.write("{:^80}\n".format("---------- List of Missing Locations ----------"))
  274. ofile.write("\n")
  275. for location in missingLocations:
  276. ofile.write(" {:<25} [{}] \n".format(location, join(idir, "%s.qsrc" % location)))
  277. #endwith
  278. with open('invalid-calls-missing-locations.md', 'w', encoding='utf-8') as ofile:
  279. ofile.write("## List of Invalid Calls\n")
  280. currLoc = ''
  281. for call in invalidLocations:
  282. if call['location'] != currLoc:
  283. currLoc = call['location']
  284. ofile.write("\n")
  285. ofile.write("### %s" % currLoc)
  286. ofile.write("\n")
  287. ofile.write('| "Function" | Is Comment |\n')
  288. ofile.write("| ---------- | ---------- |\n")
  289. ofile.write("| `%s` | %s |\n" % (call['function'], call['reason']))
  290. ofile.write("\n")
  291. ofile.write("---\n")
  292. ofile.write("\n")
  293. ofile.write("## Calls made by locations\n")
  294. currLoc = ''
  295. for call in invalidCallsMade:
  296. if call['callinglocation'] != currLoc:
  297. currLoc = call['callinglocation']
  298. ofile.write("\n")
  299. ofile.write("### %s\n" % call['callinglocation'])
  300. ofile.write("\n")
  301. ofile.write("| File | Line No. | Call | Reason |\n")
  302. ofile.write("| ---- | -------- | ---- | ------ |\n")
  303. ofile.write("| {} | {:>4} | `{} '{}', '{}'` | {} |\n".format(call['file'], call['lineNo'], call['calltype'], call['location'], call['function'], call['reason']))
  304. ofile.write("\n")
  305. ofile.write("---\n")
  306. ofile.write("\n")
  307. ofile.write("## List of Missing Locations\n")
  308. ofile.write("\n")
  309. ofile.write ("| Location | File |\n")
  310. ofile.write ("| -------- | ---- |\n")
  311. for location in missingLocations:
  312. ofile.write("| %s | %s |\n" % (location, join(idir, "%s.qsrc" % location)))
  313. #end
  314. runtime[6] = perf_counter_ns()
  315. print("%s: %d: %d" % (runtime_text[5], runtime[6], runtime[6]-runtime[5] ))
  316. print(runtime_text[6])