Faster, with some very basic "typo detection". Output still not pretty

tools/callvalidator.py

@@ -10,14 +10,255 @@ from io import open
 from time import perf_counter_ns
 import xml.etree.ElementTree as ET
 from textwrap import wrap
+from typing import List
+from threading import Thread
+from difflib import SequenceMatcher
 
-runtime = [0,0,0,0,0,0,0,0,0,0]
+
+def createcallFileList(callFileList: List) -> None:
+    validationTarget = ''
+    if len(sys.argv) > 2 and "file=" in str(sys.argv[2]):
+        validationTarget = sys.argv[2].replace("file=", "")
+        callFileList = [join(vdir, sys.argv[2].replace("file=", ""))]
+        validate = 'file'
+        
+    if len(sys.argv) > 2 and "list=" in str(sys.argv[2]):
+        validationTarget = sys.argv[2].replace("list=", "")
+        validationListFile = join(sys.argv[3].replace("folder=", ""), sys.argv[2].replace("list=", ""))
+        try:
+            tree = ET.parse(validationListFile)
+            root = tree.getroot()
+            for file in root.iter('Location'):
+                callFileList.append(join(file.attrib['folder'] if "folder" in file.attrib.keys() else idir, "%s.qsrc" % file.attrib['name'].replace('$', '_')))
+        except:
+            try: 
+                with open(validationListFile, 'r', encoding='utf-8') as ifile:
+                    lines = ifile.readlines()
+                    for index, line in enumerate(lines):
+                        temp = line.strip(" \r\n\t").split(";")
+                        file = temp[0].strip(" \t")
+                        if ".qsrc" not in file:
+                            raise SyntaxError()
+                            raise SyntaxError("Incorrect file content: '<filename>[; <folder>] expected, found %s." % line,(validationListFile, index, 1, 'if ".qsrc" not in temp[0].strip(" \t"):', index, len(line)))
+                        if len(temp) == 2:
+                            callFileList.append(join(temp[1].strip(" \t"),file))
+                        else:
+                            callFileList.append(join(idir,file))
+                    #endfor
+                #endwith
+            except SyntaxError as e:
+                raise SystemExit("Invalid list filed: %s" % e.msg)
+            #endtry   
+        #endtry  
+        validate = 'list'
+    #endif <- "list="
+
+    if validate == 'all':
+        validationTarget = idir
+        callFileList = [join(idir,f) for f in listdir(idir) if ".qsrc" in f]
+    #endif
+#enddef createValidationList()
+
+def buildValidCallsList(locationFileList) -> None:
+    # Build the list of valid calls
+    linecount = []
+    for file in locationFileList:
+        with open(join(idir, file), 'rt', encoding='utf-8', newline="\n") as ifile:
+            lines = ifile.readlines()
+            linecount.append({"file": file, "lc": len(lines)})
+            location = lines[0].strip(' #\r\n').strip(' ')
+            if location not in locationList: 
+                locationList.append(location)
+                lcLocationList.append(location.lower())
+            blockComment = False
+
+            for line in lines:
+                #getting locations
+                workline = line.strip(' \t\r\n')
+                locationsInLine = []
+                isCommentedLine = blockComment
+                
+                match = re.search(calledLinePattern, workline)
+                if match != None:
+                    potLine = re.findall(findLocationValues, workline)
+                    if not blockComment and re.search(blockCommentStart, workline) != None: 
+                        blockComment = True
+                        isCommentedLine = True
+                    if not blockComment and re.search(commentLine, workline) != None: isCommentedLine = True
+                    if potLine != None: locationsInLine = potLine
+                    if blockComment and re.search(blockCommentEnd, workline) != None: blockComment = False                    
+                #endif
+
+                for match in locationsInLine: 
+                    record = {"location": location.lower(), "function": match, "isCommentLine": isCommentedLine}
+                    recordInv = {"location": location.lower(), "function": match, "isCommentLine": not isCommentedLine}
+                    if record not in validLocationCalls and recordInv not in validLocationCalls: 
+                        validLocationCalls.append({"location": location, 
+                                                "function": match, 
+                                                "isCommentLine": isCommentedLine})
+                        lcValidLocationCalls.append({"location": location.lower(), "function": match.lower()})
+                    elif recordInv in validLocationCalls:
+                        # if commentLine of recordInv is TRUE then set it to false.
+                        # if commentLine of recordInv is FALSE then leave it alone
+                        if recordInv["isCommentLine"]:
+                            index = validLocationCalls.index(recordInv)
+                            validLocationCalls[index]['isCommentLine'] = False
+                    #endif
+                #endfor <- match in locationsInLine
+            #endfor <- lineNo, line
+        #endwith
+    #endfor
+#enddef buildValidCallsList()
+
+def validCallsByLocationTXT(validLocationCalls: List) -> None:
+    with open('valid-calls-by-location.txt', 'w', encoding='utf-8') as ofile:
+        currLoc = ''
+        ofile.write("---- List of valid calls by location\n")
+        for call in validLocationCalls:
+            if call['location'] != currLoc:
+                ofile.write("\n")
+                ofile.write("  ---- %s -------------\n" % call['location'])
+                currLoc = call['location']
+            ofile.write("    '%s': Commented Line: %s\n" % (call['function'], call['isCommentLine']))
+#enddef validCallsByLocationTXT
+
+def validCallsByLocationMD(validLocationCalls: List) -> None:
+    with open('valid-calls-by-location.md', 'w', encoding='utf-8') as ofile:
+        ofile.write("## Valid Calls per Location")
+        currLoc = ''
+        for call in validLocationCalls:
+            if call['location'] != currLoc:
+                currLoc = call['location']
+                ofile.write("\n")
+                ofile.write("### %s" % currLoc)
+                ofile.write("\n")
+                ofile.write('| "Function" | Is Comment |\n')
+                ofile.write("| ---------- | ---------- |\n")
+            ofile.write("| `'%s'` | %s |\n" % (call['function'], call['isCommentLine']))
+        ofile.write("\n")
+#enddef validCallsByLocationMD
+def validCallsForTest(validLocationCalls) -> None:
+    with open('validcallsfortesting.txt', 'w', encoding='utf-8') as ofile:
+        for call in validLocationCalls:
+            if not call['isCommentLine']: ofile.write("%s:%s\n" % (call['location'], call['function'])) 
+#enddef validCallsForTest
+
+def invalidCallsMissingLocationsTXT(invalidLocations: List, callsWithTypos: List, missingLocations: List) -> None:
+    with open('invalid-calls-missing-locations.txt', 'w', encoding='utf-8') as ofile:
+        
+        ofile.write("----- Summary ------------------------------------------\n")
+        ofile.write("  Locations called incorrectly         : {:>4}\n".format(len(invalidLocations)))
+        ofile.write("    Location/File doesn't exist        : {:>4} [{:>3.2f}%%]\n".format(invalidLocationMissing,(invalidLocationMissing/len(invalidLocations))*100))
+        ofile.write("    Commented out code                 : {:>4} [{:>3.2f}%%]\n".format(invalidCommentLine, (invalidCommentLine/len(invalidLocations))*100))
+        ofile.write("    Value not expected/handled         : {:>4} [{:>3.2f}%%]\n".format(invalidFunctionMissing, (invalidFunctionMissing/len(invalidLocations))*100))
+        ofile.write("--------------------------------------------------------\n")
+        ofile.write("  Locations making incorrect calls     : {:>4}\n".format(locationsMakingInvalidCalls))
+        ofile.write("  Total number of invalid calls        : {:>4}\n".format(len(invalidCallsMade)))
+        ofile.write("    Calls made to non-existing location: {:>4} [{:>3.2f}%%]\n".format(invalidCallsToMissingLocation, (invalidCallsToMissingLocation/len(invalidCallsMade))*100))
+        ofile.write("    Location or File name has typo     : {:>4} [{:>3.2f}%%]\n".format(invalidCallsToDueToTypos,(invalidCallsToDueToTypos/len(invalidLocations))*100))
+        ofile.write("    Calls made to commented location   : {:>4} [{:>3.2f}%%]\n".format(invalidCallsToCommentedCode, (invalidCallsToCommentedCode/len(invalidCallsMade))*100))
+        ofile.write("    Calls made with unexpected value   : {:>4} [{:>3.2f}%%]\n".format(invalidCallsToMissingFunction, (invalidCallsToMissingFunction/len(invalidCallsMade))*100))
+        ofile.write("\n")
+        ofile.write("    Valid calls with typos             : {:>4} [{:>3.2f}%%]\n".format(validCallsWithTypos, (validCallsWithTypos/(len(locationCallList)-len(invalidCallsMade)))*100))
+        ofile.write("--------------------------------------------------------\n")
+        ofile.write("  Missing locations called             : {:>4}\n".format(len(missingLocations)))
+        ofile.write("--------------------------------------------------------\n")
+        ofile.write("\n")
+        ofile.write("\n")
+        ofile.write("{:^80}\n".format("---------- List of Invalid Calls by location ----------"))
+        currLoc = ''
+        for call in invalidLocations:
+            if call['location'] != currLoc:
+                ofile.write("\n")
+                ofile.write("  ---- %s -------------\n" % call['location'])
+                currLoc = call['location']
+            wrapis = wrap("  {:<34}: {}\n".format("'%s'" % call['function'], call['reason']), 125, subsequent_indent="{:<31}".format(' '), break_long_words=True)
+            for line in wrapis:           
+                ofile.write("%s\n" % line)
+        #endfor <- call in invalidLocations
+
+        ofile.write("\n")
+        ofile.write("{:^125}\n".format("---------- Invalid Calls made by location ----------"))
+        currLoc = ''
+        for call in invalidCallsMade:
+            if call['callinglocation'] != currLoc:
+                ofile.write("\n")
+                ofile.write("  ---- %s [%s]-------------\n" % (call['callinglocation'], call['file']))
+                currLoc = call['callinglocation']
+            ofile.write("Line {:>4}: {} '{}', '{}'\n".format(call['lineNo'], call['calltype'], call['location'], call['function']))
+            for line in wrap("%s%s" % (call['reason'].replace("`", "'"), " %s" % call['typo_msg'] if call['func_typo'] else ''), 120, initial_indent='    ', subsequent_indent='    ', break_long_words=True):
+                ofile.write("%s\n"%line)
+        #endfor <- call in invalidCallsMade
+
+        ofile.write("\n")
+        ofile.write("{:^125}\n".format("---------- Calls with typos made by location ----------"))
+        currLoc = ''
+        for call in callsWithTypos:
+            if call['callinglocation'] != currLoc:
+                ofile.write("\n")
+                ofile.write("  ---- %s [%s]-------------\n" % (call['callinglocation'], call['file']))
+                currLoc = call['callinglocation']
+            ofile.write("Line {:>4}: {}\n".format(call['lineNo'], call['typo_msg']))
+        #endfor <- call in callsWithTypos
+
+        ofile.write("\n")
+        ofile.write("{:^80}\n".format("---------- List of Missing Locations ----------"))
+        
+        ofile.write("\n")
+        for location in missingLocations:
+            ofile.write("  {:<25} [{}] \n".format(location, join(idir, "%s.qsrc" % location)))
+    #endwith
+#enddef invalidCallsMissingLocationsTXT
+
+def invalidCallsMissingLocationsMD(invalidLocations: List, callsWithTypos: List, missingLocations: List) -> None:
+    with open('invalid-calls-missing-locations.md', 'w', encoding='utf-8') as ofile:
+        ofile.write("## List of Invalid Calls\n")    
+        currLoc = ''
+        for call in invalidLocations:            
+            if call['location'] != currLoc:
+                currLoc = call['location']
+                ofile.write("\n")
+                ofile.write("### %s" % currLoc)
+                ofile.write("\n")
+                ofile.write('| "Function" | Is Comment |\n')
+                ofile.write("| ---------- | ---------- |\n")
+            ofile.write("| `%s` | %s |\n" % (call['function'], call['reason']))
+        ofile.write("\n")
+        ofile.write("---\n")
+        ofile.write("\n")
+        ofile.write("## Calls made by locations\n")
+        currLoc = ''
+        for call in invalidCallsMade:
+            if call['callinglocation'] != currLoc:
+                currLoc = call['callinglocation']
+                ofile.write("\n")
+                ofile.write("### %s\n" % call['callinglocation'])
+                ofile.write("\n")
+                ofile.write("| File | Line No. | Call | Reason |\n")
+                ofile.write("| ---- | -------- | ---- | ------ |\n")
+            ofile.write("| {} | {:>4} | `{} '{}', '{}'` | {} |\n".format(call['file'], call['lineNo'], call['calltype'], call['location'], call['function'], call['reason']))
+        ofile.write("\n")
+        ofile.write("---\n")
+        ofile.write("\n")
+        ofile.write("## List of Missing Locations\n")
+        ofile.write("\n")
+        ofile.write ("| Location | File |\n")
+        ofile.write ("| -------- | ---- |\n")
+        for location in missingLocations:
+            ofile.write("| %s | %s |\n" % (location, join(idir, "%s.qsrc" % location)))
+    #endwith
+#enddef invalidCallsMissingLocationsMD
+
+
+runtime = [0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 runtime[0] = perf_counter_ns()
 
 idir = ''
 validLocationCalls = []
-locationFileList = []
+lcValidLocationCalls = []
 locationList = []
+lcLocationList = []
+locationFileList = []
 missingLocations = []
 locationCallList = []
 callFileList = []
@@ -34,9 +275,24 @@ callLinePattern = "(gt|gs|xgt|xgs)(?:\s*(?:'|\"))(\w+)(?:'|\")(?:(?:,\s*)(?:'|\"
 findCallLine = "(?:^\s*!+.*{)|(?:^[^!]*)(gt|gs|xgt|xgs)\s*(?:'|\")(\w+)(?:'|\")(?:(?:,\s*)(?:'|\")(\w+)(?:'|\"))?|(?:}[^{]*$)"
 calledLinePattern = "(?:^\s*!+.*{)|(?:(?:if|elseif)?\s*\(?\$?(?:ARGS|args)\[(?:0|i)\])(?:\s*(?:=|!|<|>|>=|=>|<=|=<)\s*(?:'|\"))(\w+)(?:'|\")|(?:}[^{]*$)"
 findLocationValues = "(?:(?:if|elseif)?\s*\(?\$?(?:ARGS|args)\[(?:0|i)\])(?:\s*(?:=|!|<|>|>=|=>|<=|=<)\s*(?:'|\"))(\w+)(?:'|\")"
+runtime_text = ['Initialising "global" variables',                      #0
+                "Initialise with Arguments",                            #1
+                "Build call file list and valid calls list (%d files)", #2
+                "Building call list [%d files]",                        #3
+                "Sorting and ordering the lists for the files",         #4
+                "Creating error type list [invalidLocations]",          #5
+                "Creating error type list [callsWithTypos]",            #6
+                "Generating file output",                               #7
+                "All is finished",                                      #8
+                "",                                                     #9
+                "",                                                     #10
+                "",                                                     #11
+                "",                                                     #12
+                ""]                                                     #13
 
-runtime_text = ["Setup started","Started callfile build","","","Sorting and ordering the lists for the files","","All is finished"]
 
+runtime[1] =  perf_counter_ns()
+print("0. %s: %d : %d: %d" % (runtime_text[0], runtime[0], runtime[1], runtime[1]-runtime[0] ))
 assert (len(sys.argv) == 2 or len(sys.argv) == 3 or len(sys.argv) == 4), "usage:\ncallvalidator.py source=<src_input_dir> [file=<filename> | list=<listfilename>] [folder=<folder>]"
 
 idir = str(sys.argv[1].replace("source=", ""))
@@ -46,100 +302,23 @@ if len(sys.argv) == 4:
 else:
     vdir = idir
 
-validationTarget = ''
-runtime[1] = perf_counter_ns()
-print("%s: %d: %d" % (runtime_text[0], runtime[1], runtime[1]-runtime[0] ))
-if len(sys.argv) > 2 and "file=" in str(sys.argv[2]):
-    validationTarget = sys.argv[2].replace("file=", "")
-    callFileList = [join(vdir, sys.argv[2].replace("file=", ""))]
-    validate = 'file'
-    
-if len(sys.argv) > 2 and "list=" in str(sys.argv[2]):
-    validationTarget = sys.argv[2].replace("list=", "")
-    validationListFile = join(sys.argv[3].replace("folder=", ""), sys.argv[2].replace("list=", ""))
-    try:
-        tree = ET.parse(validationListFile)
-        root = tree.getroot()
-        for file in root.iter('Location'):
-            callFileList.append(join(file.attrib['folder'] if "folder" in file.attrib.keys() else idir, "%s.qsrc" % file.attrib['name'].replace('$', '_')))
-    except:
-        try: 
-            with open(validationListFile, 'r', encoding='utf-8') as ifile:
-                lines = ifile.readlines()
-                for index, line in enumerate(lines):
-                    temp = line.strip(" \r\n\t").split(";")
-                    file = temp[0].strip(" \t")
-                    if ".qsrc" not in file:
-                        raise SyntaxError()
-                        raise SyntaxError("Incorrect file content: '<filename>[; <folder>] expected, found %s." % line,(validationListFile, index, 1, 'if ".qsrc" not in temp[0].strip(" \t"):', index, len(line)))
-                    if len(temp) == 2:
-                        callFileList.append(join(temp[1].strip(" \t"),file))
-                    else:
-                        callFileList.append(join(idir,file))
-                #endfor
-            #endwith
-        except SyntaxError as e:
-            raise SystemExit("Invalid list filed: %s" % e.msg)
-        #endtry   
-    #endtry  
-    validate = 'list'
-#endif <- "list="
-
-if validate == 'all':
-    validationTarget = idir
-    callFileList = [join(idir,f) for f in listdir(idir) if ".qsrc" in f]
-#endif
-
-# Build the list of valid calls
 locationFileList = [f for f in listdir(idir) if ".qsrc" in f and f not in fileToIgnore]
+
+callFileListThread = Thread(target=createcallFileList(callFileList), args=(callFileList,))
+validCallsListThread = Thread(buildValidCallsList(locationFileList), args=(locationFileList))
+
 runtime[2] = perf_counter_ns()
-runtime_text[2] = "Building valid calls list [%d files]" % len(locationFileList)
-print("%s: %d: %d" % (runtime_text[1], runtime[2], runtime[2]-runtime[1] ))
-for file in locationFileList:
-    with open(join(idir, file), 'rt', encoding='utf-8', newline="\n") as ifile:
-        lines = ifile.readlines()
-        location = lines[0].strip(' #\r\n').strip(' ')
-        if location not in locationList: locationList.append(location)
-        blockComment = False
-        for lineNo, line in enumerate(lines):
-            #getting locations
-            workline = line.strip(' \t\r\n')
-            locationsInLine = []
-            isCommentedLine = blockComment
-            match = re.search(calledLinePattern, workline)
-            if match != None:
-                potLine = re.findall(findLocationValues, workline)
-                if not blockComment and re.search(blockCommentStart, workline) != None: 
-                    blockComment = True
-                    isCommentedLine = True
-                if not blockComment and re.search(commentLine, workline) != None: isCommentedLine = True
-                if potLine != None: locationsInLine = potLine
-                if blockComment and re.search(blockCommentEnd, workline) != None: blockComment = False                    
-            #endif
+print("1. %s: %d : %d: %d" % (runtime_text[1], runtime[1], runtime[2], runtime[2]-runtime[1] ))
 
-            for match in locationsInLine: 
-                record = {"location": location, "function": match, "isCommentLine": isCommentedLine}
-                recordInv = {"location": location, "function": match, "isCommentLine": not commentLine}
-                if record not in validLocationCalls and recordInv not in validLocationCalls: 
-                    validLocationCalls.append({"location": location, 
-                                            "function": match, 
-                                            "isCommentLine": isCommentedLine})
-                elif recordInv in validLocationCalls:
-                    # if commentLine of recordInv is TRUE then set it to false.
-                    # if commentLine of recordInv is FALSE then leave it alone
-                    if recordInv["isCommentLine"]:
-                        index = validLocationCalls.index(recordInv)
-                        validLocationCalls[index]['isCommentLine'] = False
-                #endif
-            #endfor <- match in locationsInLine
-        #endfor <- lineNo, line
-    #endwith
-#endfor
+callFileListThread.start()
+validCallsListThread.start()
 
-# build a list of all the calls happening
+callFileListThread.join()
+validCallsListThread.join()
 runtime[3] = perf_counter_ns()
-runtime_text[3] = "Building call list [%d files]" % len(callFileList)  
-print("%s: %d: %d" % (runtime_text[2], runtime[3], runtime[3]-runtime[2] ))
+print("2. %s: %d : %d: %d" % (runtime_text[2] % len(locationFileList), runtime[2], runtime[3], runtime[3]-runtime[2] ))
+
+# build a list of all the calls happening
 for file in callFileList:
     with open(file, 'rt', encoding='utf-8', newline="\n") as ifile:
         lines = ifile.readlines()
@@ -148,6 +327,7 @@ for file in callFileList:
         for lineNo, line in enumerate(lines):
             workline = line.strip(' \t\n\r')
             match = None
+            msg = ''
             temp_match = re.search(findCallLine, workline)
             if temp_match != None and not [res for res in locationsToIgnore if res in workline.lower()]:
                 potLine = re.search(callLinePattern, workline)
@@ -158,66 +338,125 @@ for file in callFileList:
             
             if match != None:
                 valid = 0
-                reason = ""
-                searchRecordF = {"location": match.group(2), "function": '' if match.group(3) == None else match.group(3), "isCommentLine": False}
-                searchRecordT = {"location": match.group(2), "function": '' if match.group(3) == None else match.group(3), "isCommentLine": True}
-                if searchRecordF['function'] == '':
-                    if searchRecordF not in validLocationCalls: validLocationCalls.append(searchRecordF)
-                    valid = 1
-                    reason = ""
-                elif searchRecordF['location'] not in locationList:
+                func_typo = 0
+                loc_typo = 0
+                typo_msg = ""
+                loc_msg = ""
+                func_msg = "."
+                reason = ""            
+                loc = match.group(2)   
+                func = '' if match.group(3) == None else match.group(3) 
+                searchRecord = {"location": loc.lower(), "function": func.lower()}
+                if searchRecord['location'] not in lcLocationList:
                     valid = -1
-                    reason = "Location `%s` doesn't exist - No file containing this location was found in the specified folder." % (searchRecordF['location'])
-                    if searchRecordF['location'] not in missingLocations: missingLocations.append(searchRecordF['location'])
-                elif searchRecordT in validLocationCalls:
-                    valid = -2
-                    reason = "`%s, %s` exists but is either commented out or is in a comment block" % (searchRecordT['location'], searchRecordT['function'])
-                elif searchRecordF in validLocationCalls:
-                    valid = 1
-                    reason = ""
+                    reason = "Location `%s` doesn't exist - No file containing this location was found in the specified folder." % (loc)
+                    if loc not in missingLocations: missingLocations.append(loc)
                 else:
-                    valid = -3 
-                    reason = "Location `%s` exists but couldn't find any entrypoint expecting `%s` as $ARGS[0]. Please confirm whether this is a bug or not." % (searchRecordF['location'], searchRecordF['function'])
+                    if loc not in locationList:
+                        index = lcLocationList.index(searchRecord['location'])
+                        loc_typo = 1
+                        loc_msg = "The location name is '%s' not '%s'" % (locationList[index], loc) 
+                    #endif
+                    if searchRecord['function'] == '' and searchRecord not in lcValidLocationCalls:
+                        validLocationCalls.append({"location": loc, "function": '', "isCommentLine": False })
+                        lcValidLocationCalls.append(searchRecord)
+                    elif searchRecord in lcValidLocationCalls:
+                        index = lcValidLocationCalls.index(searchRecord)
+                        validL = validLocationCalls[index]
+                        if validL['function'] == func:
+                            if not validL['isCommentLine']:
+                                valid = 1
+                                reason = ""
+                            else:
+                                valid = -2
+                                reason = "`%s, %s` exists but is either commented out or is in a comment block" % (loc, func)                    
+                            #endif
+                        else:
+                            valid = -3
+                            reason = "Location `%s` exists but couldn't find any entrypoint expecting `%s` as $ARGS[0]. Please confirm whether this is a bug or not." % (loc, func)
+                            func_typo = 1
+                            if validL['isCommentLine']: 
+                                if loc_typo:
+                                    func_msg = " and $AGRS[0] has a case typo, it is '%s' not '%s'. Also, the correct call is a comment line or in a comment block." % (validL['function'], func)
+                                else:
+                                    func_msg = "$AGRS[0] has a case typo, it should be '%s' not '%s'. Also, the correct call is a comment line or in a comment block." % (validL['function'], func)
+                            else:
+                                if loc_typo:
+                                    func_msg = " and $AGRS[0] has a case typo, it is '%s' not '%s'." % (validL['function'], func)
+                                else:
+                                    func_msg = "$AGRS[0] has a case typo, it should be '%s' not '%s'." % (validL['function'], func)
+                            #endif
+                        #endif
+                    else:              
+                        strictness = 0.75       
+                        possible_funcs = []
+                        for f in (f['function'] for f in validLocationCalls if f['location'] == loc.lower() and 
+                                  f['function'] not in possible_funcs and 
+                                  f['function'] != func and 
+                                  ("%s_" % func) not in f['function'] and 
+                                  ("%s_" % f['function']) not in func):
+                            similarity = SequenceMatcher(None, func, f, strictness)
+                            if similarity.ratio() >= strictness:
+                                possible_funcs.append("'%s'" % f)
+
+                        valid = -3
+                        reason = "Location `%s` exists but couldn't find any entrypoint expecting `%s` as $ARGS[0]. Please confirm whether this is a bug or not." % (loc, func)
+                        if len(possible_funcs) > 0:
+                            reason += "\nThere may be a typo in the $ARGS[0] value passed. Some similar expected $ARGS[0] values are: %s" % ", ".join(possible_funcs)
+                    #endif
                 #endif    
+                if loc_typo or func_typo: typo_msg = "The call has %s. %s%s [%s]" % ("a typo" if func_typo != loc_typo else "typos", loc_msg, func_msg, "The call will fail" if func_typo else "The call will work")
 
                 locationCallList.append({"callinglocation": location, 
                                         "file": ifile.name, 
                                         "lineNo": lineNo+1, 
                                         "calltype": match.group(1), 
-                                        "location": match.group(2), 
-                                        "function": '' if match.group(3) == None else match.group(3), 
+                                        "location": loc, 
+                                        "function": func, 
                                         "valid": valid, 
-                                        "reason": reason})    
+                                        "reason": reason,
+                                        "loc_typo": loc_typo,
+                                        "func_typo": func_typo,
+                                        "typo_msg": typo_msg})   
+                i = 0 
             #endif# <- Match != None
         #endfor <- line in lines
     #endwith
 #endfor
 
 runtime[4] = perf_counter_ns()
-print("%s: %d: %d" % (runtime_text[3], runtime[4], runtime[4]-runtime[3] ))
+print("3. %s: %d : %d: %d" % (runtime_text[3] % len(callFileList), runtime[3],runtime[4], runtime[4]-runtime[3] ))
 
 missingLocations = sorted(missingLocations)
 validLocationCalls = sorted(validLocationCalls, key=lambda k: (k['location'].lower(), k['function']))
 invalidCallsMade = [call for call in locationCallList if call['valid'] < 0]
 invalidCallsMade = sorted(invalidCallsMade, key = lambda k: (k['callinglocation'].lower(), k['lineNo']))
 invalidLocations = []
+callsWithTypos = []
 invalidCallsToMissingLocation = 0
 invalidCallsToCommentedCode = 0
 invalidCallsToMissingFunction = 0
+invalidCallsToDueToTypos = 0
 invalidLocationMissing = 0
 invalidCommentLine = 0
 invalidFunctionMissing = 0
 locationsMakingInvalidCalls = 0
+validCallsWithTypos = 0
+
+runtime[5] = perf_counter_ns()
+print("4. %s: %d : %d: %d" % (runtime_text[4], runtime[4], runtime[5], runtime[5]-runtime[4] ))
 
 currLoc = ''
 for call in invalidCallsMade:
-    record = {"location": call['location'], "function": call['function'], "valid": call['valid'], "reason": call['reason']}
+    record = {"location": call['location'], "function": call['function'], "valid": call['valid'], "reason": call['reason'], "func_type": call['func_typo'], "loc_typo": call['loc_typo'], "typo_msg": call['typo_msg']}
     if call['callinglocation'] != '':
         locationsMakingInvalidCalls += 1
         currLoc = call['callinglocation']
     if call['valid'] == -1: invalidCallsToMissingLocation += 1
     if call['valid'] == -2: invalidCallsToCommentedCode += 1
     if call['valid'] == -3: invalidCallsToMissingFunction += 1
+    if call['func_typo']  : invalidCallsToDueToTypos += 1
+
     if record not in invalidLocations: 
         invalidLocations.append(record)
         if record['valid'] == -1: invalidLocationMissing += 1
@@ -225,126 +464,35 @@ for call in invalidCallsMade:
         if record['valid'] == -3: invalidFunctionMissing += 1
 invalidLocations = sorted(invalidLocations, key=lambda k: (k['location'].lower(), k['function'])) 
 
-runtime[5] = perf_counter_ns()
-print("%s: %d: %d" % (runtime_text[4], runtime[5], runtime[5]-runtime[4] ))
-
-with open('valid-calls-by-location.txt', 'w', encoding='utf-8') as ofile:
-    currLoc = ''
-    ofile.write("---- List of valid calls by location\n")
-    for call in validLocationCalls:
-        if call['location'] != currLoc:
-            ofile.write("\n")
-            ofile.write("  ---- %s -------------\n" % call['location'])
-            currLoc = call['location']
-        ofile.write("    '%s': Commented Line: %s\n" % (call['function'], call['isCommentLine']))
-
-with open('validcallsfortesting.txt', 'w', encoding='utf-8') as ofile:
-    for call in validLocationCalls:
-        if not call['isCommentLine']: ofile.write("%s:%s\n" % (call['location'], call['function'])) 
-
-with open('valid-calls-by-location.md', 'w', encoding='utf-8') as ofile:
-    ofile.write("## Valid Calls per Location")
-    currLoc = ''
-    for call in validLocationCalls:
-        if call['location'] != currLoc:
-            currLoc = call['location']
-            ofile.write("\n")
-            ofile.write("### %s" % currLoc)
-            ofile.write("\n")
-            ofile.write('| "Function" | Is Comment |\n')
-            ofile.write("| ---------- | ---------- |\n")
-        ofile.write("| `'%s'` | %s |\n" % (call['function'], call['isCommentLine']))
-    ofile.write("\n")
-
-with open('invalid-calls-missing-locations.txt', 'w', encoding='utf-8') as ofile:
-    
-    ofile.write("----- Summary ------------------------------------------\n")
-    ofile.write("  Locations called incorrectly         : {:>3}\n".format(len(invalidLocations)))
-    ofile.write("    Location/File doesn't exist        : {:>3} [{:>3.2f}%%]\n".format(invalidLocationMissing,(invalidLocationMissing/len(invalidLocations))*100))
-    ofile.write("    Commented out code                 : {:>3} [{:>3.2f}%%]\n".format(invalidCommentLine, (invalidCommentLine/len(invalidLocations))*100))
-    ofile.write("    Value not expected/handled         : {:>3} [{:>3.2f}%%]\n".format(invalidFunctionMissing, (invalidFunctionMissing/len(invalidLocations))*100))
-    ofile.write("--------------------------------------------------------\n")
-    ofile.write("  Locations making incorrect calls     : {:>3}\n".format(locationsMakingInvalidCalls))
-    ofile.write("  Total number of invalid calls        : {:>3}\n".format(len(invalidCallsMade)))
-    ofile.write("    Calls made to non-existing location: {:>3} [{:>3.2f}%%]\n".format(invalidCallsToMissingLocation, (invalidCallsToMissingLocation/len(invalidCallsMade))*100))
-    ofile.write("    Calls made to commented location   : {:>3} [{:>3.2f}%%]\n".format(invalidCallsToCommentedCode, (invalidCallsToCommentedCode/len(invalidCallsMade))*100))
-    ofile.write("    Calls made with unexpected value   : {:>3} [{:>3.2f}%%]\n".format(invalidCallsToMissingFunction, (invalidCallsToMissingFunction/len(invalidCallsMade))*100))
-    ofile.write("--------------------------------------------------------\n")
-    ofile.write("  Missing locations called             : {:>3}\n".format(len(missingLocations)))
-    ofile.write("--------------------------------------------------------\n")
-    ofile.write("\n")
-    ofile.write("\n")
-    ofile.write("{:^80}\n".format("---------- List of Invalid Calls by location ----------"))
-    currLoc = ''
-    for call in invalidLocations:
-        if call['location'] != currLoc:
-            ofile.write("\n")
-            ofile.write("  ---- %s -------------\n" % call['location'])
-            currLoc = call['location']
-        wrapis = wrap("  {:<34}: {}\n".format("'%s'" % call['function'], call['reason']), 125, subsequent_indent="{:<31}".format(' '), break_long_words=True)
-        for line in wrapis:           
-            ofile.write("%s\n" % line)
-    #endfor <- call in invalidLocations
-
-    ofile.write("\n")
-    ofile.write("{:^125}\n".format("---------- Calls made by location ----------"))
-    currLoc = ''
-    for call in invalidCallsMade:
-        if call['callinglocation'] != currLoc:
-            ofile.write("\n")
-            ofile.write("  ---- %s [%s]-------------\n" % (call['callinglocation'], call['file']))
-            currLoc = call['callinglocation']
-        ofile.write("Line {:>4}: {} '{}', '{}'\n".format(call['lineNo'], call['calltype'], call['location'], call['function']))
-        for line in wrap(call['reason'].replace("`", "'"), 90, initial_indent='    ', subsequent_indent='    ', break_long_words=True):
-            ofile.write("%s\n"%line)
-    #endfor <- call in invalidCallsMade
-
-    ofile.write("\n")
-    ofile.write("{:^80}\n".format("---------- List of Missing Locations ----------"))
-    
-    ofile.write("\n")
-    for location in missingLocations:
-        ofile.write("  {:<25} [{}] \n".format(location, join(idir, "%s.qsrc" % location)))
-#endwith
-
-with open('invalid-calls-missing-locations.md', 'w', encoding='utf-8') as ofile:
-    ofile.write("## List of Invalid Calls\n")    
-    currLoc = ''
-    for call in invalidLocations:            
-        if call['location'] != currLoc:
-            currLoc = call['location']
-            ofile.write("\n")
-            ofile.write("### %s" % currLoc)
-            ofile.write("\n")
-            ofile.write('| "Function" | Is Comment |\n')
-            ofile.write("| ---------- | ---------- |\n")
-        ofile.write("| `%s` | %s |\n" % (call['function'], call['reason']))
-    ofile.write("\n")
-    ofile.write("---\n")
-    ofile.write("\n")
-    ofile.write("## Calls made by locations\n")
-    currLoc = ''
-    for call in invalidCallsMade:
-        if call['callinglocation'] != currLoc:
-            currLoc = call['callinglocation']
-            ofile.write("\n")
-            ofile.write("### %s\n" % call['callinglocation'])
-            ofile.write("\n")
-            ofile.write("| File | Line No. | Call | Reason |\n")
-            ofile.write("| ---- | -------- | ---- | ------ |\n")
-        ofile.write("| {} | {:>4} | `{} '{}', '{}'` | {} |\n".format(call['file'], call['lineNo'], call['calltype'], call['location'], call['function'], call['reason']))
-    ofile.write("\n")
-    ofile.write("---\n")
-    ofile.write("\n")
-    ofile.write("## List of Missing Locations\n")
-    ofile.write("\n")
-    ofile.write ("| Location | File |\n")
-    ofile.write ("| -------- | ---- |\n")
-    for location in missingLocations:
-        ofile.write("| %s | %s |\n" % (location, join(idir, "%s.qsrc" % location)))
-#end
-
 runtime[6] = perf_counter_ns()
-print("%s: %d: %d" % (runtime_text[5], runtime[6], runtime[6]-runtime[5] ))
-print(runtime_text[6])
+print("5. %s: %d : %d: %d" % (runtime_text[5], runtime[5], runtime[6], runtime[6]-runtime[5] ))
+
+##endfor for call in locationCallList
+callsWithTypos = [call for call in locationCallList if call['loc_typo'] or call['func_typo']]
+validCallsWithTypos = len(callsWithTypos)
+callsWithTypos = sorted(callsWithTypos, key=lambda k: (k['callinglocation'].lower(), k['lineNo'])) 
+
+runtime[7] = perf_counter_ns()
+print("6. %s: %d : %d: %d" % (runtime_text[6], runtime[6], runtime[7], runtime[7]-runtime[6] ))
+
+fileThread1 = Thread(target=validCallsByLocationTXT(validLocationCalls), args=(validLocationCalls,))
+#fileThread2 = Thread(target=validCallsByLocationMD(validLocationCalls), args=(validLocationCalls,))
+fileThread3 = Thread(target=validCallsForTest(validLocationCalls), args=(validLocationCalls,))
+fileThread4 = Thread(target=invalidCallsMissingLocationsTXT(invalidLocations, callsWithTypos, missingLocations), args=(invalidLocations, callsWithTypos, missingLocations))
+#fileThread5 = Thread(target=invalidCallsMissingLocationsMD(invalidLocations, callsWithTypos, missingLocations), args=(invalidLocations, callsWithTypos, missingLocations))
+
+fileThread1.start()
+#fileThread2.start()
+fileThread3.start()
+fileThread4.start()
+#fileThread5.start()
+
+fileThread1.join()
+#fileThread2.join()
+fileThread3.join()
+fileThread4.join()
+#fileThread5.join()
 
+runtime[8] = perf_counter_ns()
+print("7. %s: %d : %d: %d" % (runtime_text[7], runtime[7], runtime[8], runtime[8]-runtime[7] ))
+print("8. %s: %d : %d: %d" % (runtime_text[8], runtime[0], runtime[8], runtime[8]-runtime[0] ))