Refactored the extraction script for easier integration of new images/symbols

Offsets/CiOffsets.csv

@@ -177,6 +177,7 @@ ci_17763-4644.dll,36d58,4bb30
 ci_17763-4737.dll,36d58,4bb30
 ci_17763-4840.dll,36d58,4bb30
 ci_17763-4974.dll,36d58,4bb30
+ci_17763-5122.dll,36d58,4bb30
 ci_17763-10458.dll,36d18,4ba70
 ci_17763-10877.dll,36d18,4bae0
 ci_18362-1.dll,37278,4c600

Offsets/ExtractOffsets.py

@@ -17,8 +17,12 @@ THREADS_LIMIT = None
 CSVLock = threading.Lock()
 
 machineType = dict(x86=332, x64=34404)
-knownImageVersions = dict(ntoskrnl=list(), wdigest=list(), ci=list())
+supported_images = ["ntoskrnl.exe", "wdigest.dll", "ci.dll"]
-extensions_by_mode = dict(ntoskrnl="exe", wdigest="dll", ci="dll")
+modes = [image_name.split(".")[0] for image_name in supported_images]
+extensions_by_mode = dict(image_name.split(".") for image_name in supported_images)
+known_image_versions = {mode: list() for mode in modes}
+modes_by_imagename = dict(zip(supported_images, modes))
+csvFilenameByMode = {mode: mode.capitalize() + "Offsets.csv" for mode in modes}
 
 symbols = dict(
     ntoskrnl=[
@@ -44,6 +48,8 @@ symbols = dict(
     ],
 )
 
+symbols_names = {mode: [t[0] if t[-1] == "symbol" else f"{t[0]}_{t[1]}" for t in symbols[mode]] for mode in modes}
+
 
 def find(key: str, d: dict):
     for k, v in d.items():
@@ -252,12 +258,10 @@ def extractOffsets(input_file, output_file, mode):
             export_directory_rva = export_directory_entry.VirtualAddress
             image_name_rva = pe.get_dword_at_rva(export_directory_rva + 3 * 4)
             name = pe.get_string_at_rva(image_name_rva).decode().lower()
-            if "ntoskrnl.exe" in name:
+            for image_name in supported_images:
-                imageType = "ntoskrnl"
+                if image_name in name:
-            elif "wdigest.dll" in name:
+                    imageType = modes_by_imagename[image_name]
-                imageType = "wdigest"
+                    break
-            elif "ci.dll" in name:
-                imageType = "ci"
             else:
                 print(f"[*] File {input_file} unrecognized")
                 return
@@ -274,7 +278,7 @@ def extractOffsets(input_file, output_file, mode):
             extension = extensions_by_mode[imageType]
             imageVersion = f"{imageType}_{full_version[2]}-{full_version[3]}.{extension}"
 
-            if imageVersion in knownImageVersions[imageType]:
+            if imageVersion in known_image_versions[imageType]:
                 print(f"[*] Skipping known {imageType} version {imageVersion} (file: {input_file})")
                 try:
                     """
@@ -296,7 +300,7 @@ def extractOffsets(input_file, output_file, mode):
                         for part in input_file_basename[len(f"{imageType}_") : -len(f".{extension}")].split("-")
                     )
                     imageVersion = input_file_basename
-                    if imageVersion in knownImageVersions[imageType]:
+                    if imageVersion in known_image_versions[imageType]:
                         return
                     print("\r", end="")  # Not skipping after all
                 except ValueError:
@@ -330,7 +334,7 @@ def extractOffsets(input_file, output_file, mode):
 
             # print("wrote into CSV !")
             del pdb
-            knownImageVersions[imageType].append(imageVersion)
+            known_image_versions[imageType].append(imageVersion)
             print(f"[+] Finished processing of {imageType} {input_file}!")
 
         except PEFormatError as e:
@@ -381,22 +385,25 @@ def sortOutputFile(csvFile):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
 
+    modes_str = "/".join(known_image_versions)
+    files = " / ".join(modes_by_imagename)
+    csvfiles = " / ".join(csvFilenameByMode.values())
     parser.add_argument(
         "mode",
-        help='"ntoskrnl", "wdigest" or "ci". Mode to download and extract offsets from either ntoskrnl.exe, wdigest.dll or ci.dll',
+        help=f"{modes_str}. Mode to download and extract offsets from either {files}",
     )
     parser.add_argument(
         "-i",
         "--input",
         dest="input",
         required=True,
-        help="Single file or directory containing ntoskrnl.exe / wdigest.dll / ci.dll to extract offsets from. If in download mode, the PE downloaded from MS symbols servers will be placed in this folder.",
+        help=f"Single file or directory containing {files} to extract offsets from. If in download mode, the PE downloaded from MS symbols servers will be placed in this folder.",
     )
     parser.add_argument(
         "-o",
         "--output",
         dest="output",
-        help="CSV file to write offsets to. If the specified file already exists, only new ntoskrnl versions will be downloaded / analyzed. Defaults to NtoskrnlOffsets.csv / WdigestOffsets.csv / CiOffsets.csv in the current folder.",
+        help=f"CSV file to write offsets to. If the specified file already exists, only new ntoskrnl versions will be downloaded / analyzed. Defaults to {csvfiles} in the current folder.",
     )
     parser.add_argument(
         "-d",
@@ -408,20 +415,20 @@ if __name__ == "__main__":
 
     args = parser.parse_args()
     mode = args.mode.lower()
-    if mode not in knownImageVersions:
+    if mode not in known_image_versions:
-        print(f'[!] ERROR : unsupported mode "{args.mode}", supported mode are: "ntoskrnl", "wdigest" and "ci"')
+        print(f'[!] ERROR : unsupported mode "{args.mode}", supported mode are: {modes}')
         exit(1)
 
     # If the output file exists, load the already analyzed image versions.
     # Otherwise, write CSV headers to the new file.
     if not args.output:
-        args.output = mode.capitalize() + "Offsets.csv"
+        args.output = csvFilenameByMode[mode]
     if os.path.isfile(args.output):
-        loadOffsetsFromCSV(knownImageVersions[mode], args.output)
+        loadOffsetsFromCSV(known_image_versions[mode], args.output)
-        print(f'[+] Loaded {len(knownImageVersions[mode])} known {mode} versions from "{args.output}"')
+        print(f'[+] Loaded {len(known_image_versions[mode])} known {mode} versions from "{args.output}"')
     else:
         with open(args.output, "w") as output:
-            output.write(mode + "Version," + ",".join(elem[0] for elem in symbols[mode]) + "\n")
+            output.write(mode + "Version," + ",".join(elem for elem in symbols_names[mode]) + "\n")
 
     # In download mode, an updated list of image versions published will be retrieved from https://winbindex.m417z.com.
     # The symbols for each version will be downloaded from the Microsoft symbols servers.
@@ -431,7 +438,7 @@ if __name__ == "__main__":
             print("[!] ERROR : in download mode, -i / --input option must specify a folder")
             exit(1)
         extension = extensions_by_mode[mode]
-        downloadPEFileFromMS(mode, extension, knownImageVersions[mode], args.input)
+        downloadPEFileFromMS(mode, extension, known_image_versions[mode], args.input)
 
     # Extract the offsets from the specified file or the folders containing image files.
     extractOffsets(args.input, args.output, mode)

Offsets/NtoskrnlOffsets.csv

@@ -1,4 +1,4 @@
-ntoskrnlVersion,PspCreateProcessNotifyRoutine,PspCreateThreadNotifyRoutine,PspLoadImageNotifyRoutine,_EPROCESS,EtwThreatIntProvRegHandle,_ETW_REG_ENTRY,_ETW_GUID_ENTRY,PsProcessType,PsThreadType,_OBJECT_TYPE,SeCiCallbacks
+ntoskrnlVersion,PspCreateProcessNotifyRoutine,PspCreateThreadNotifyRoutine,PspLoadImageNotifyRoutine,_EPROCESS_Protection,EtwThreatIntProvRegHandle,_ETW_REG_ENTRY_GuidEntry,_ETW_GUID_ENTRY_ProviderEnableInfo,PsProcessType,PsThreadType,_OBJECT_TYPE_CallbackList,SeCiCallbacks
 ntoskrnl_10240-16384.exe,35d2e0,35d0e0,35cee0,6aa,0,20,50,3c51e8,3c5200,c8,31ee80
 ntoskrnl_10240-17394.exe,35d420,35d220,35d020,6aa,0,20,50,3c51e8,3c5200,c8,31ef40
 ntoskrnl_10240-17443.exe,35c420,35c220,35c020,6aa,0,20,50,3c41e8,3c4200,c8,31df40
@@ -55,6 +55,7 @@ ntoskrnl_10240-20048.exe,369520,369320,369120,6b2,0,20,50,3cf230,3cf248,c8,32b06
 ntoskrnl_10240-20107.exe,3695a0,3693a0,3691a0,6b2,0,20,50,3cf228,3cf248,c8,32b0a0
 ntoskrnl_10240-20161.exe,369560,369360,369160,6b2,0,20,50,3cf228,3cf248,c8,32b060
 ntoskrnl_10240-20232.exe,369560,369360,369160,6b2,0,20,50,3cf228,3cf248,c8,32b060
+ntoskrnl_10240-20307.exe,369560,369360,369160,6b2,0,20,50,3cf228,3cf248,c8,32b060
 ntoskrnl_10586-0.exe,317180,316f80,316d80,6b2,0,20,50,37f228,37f248,c8,2d8d40
 ntoskrnl_10586-1176.exe,3161c0,315fc0,315dc0,6b2,0,20,50,37e228,37e248,c8,2d7d00
 ntoskrnl_10586-1177.exe,3161c0,315fc0,315dc0,6b2,0,20,50,37e228,37e248,c8,2d7d00
@@ -154,6 +155,7 @@ ntoskrnl_14393-5921.exe,33ce20,33cc20,33ca20,6ca,0,20,50,3a9250,3a9278,c8,2fffa0
 ntoskrnl_14393-5996.exe,33cf20,33cd20,33cb20,6ca,0,20,50,3a9250,3a9278,c8,300080
 ntoskrnl_14393-6085.exe,33cea0,33cca0,33caa0,6ca,0,20,50,3a9250,3a9278,c8,300020
 ntoskrnl_14393-6167.exe,33ce60,33cc60,33ca60,6ca,0,20,50,3a9250,3a9278,c8,300020
+ntoskrnl_14393-6451.exe,33cea0,33cca0,33caa0,6ca,0,20,50,3a9250,3a9278,c8,300040
 ntoskrnl_15063-0.exe,382290,382090,381e90,6ca,341ea8,20,50,3e1f98,3e1fb0,c8,345be0
 ntoskrnl_15063-13.exe,382290,382090,381e90,6ca,341ea8,20,50,3e1f98,3e1fb0,c8,345be0
 ntoskrnl_15063-296.exe,382290,382090,381e90,6ca,341ea8,20,50,3e1f98,3e1fb0,c8,345be0
@@ -433,6 +435,7 @@ ntoskrnl_17763-4644.exe,4d8900,4d8b00,4d8700,6ca,409458,20,60,5402d0,5402f8,c8,4
 ntoskrnl_17763-4737.exe,4d8940,4d8b40,4d8740,6ca,409478,20,60,5412d0,5412f8,c8,40cc40
 ntoskrnl_17763-4851.exe,4d8c00,4d8800,4d8a00,6ca,4094b8,20,60,5412d0,5412f8,c8,40cca0
 ntoskrnl_17763-4974.exe,4d8b40,4d8740,4d8940,6ca,409478,20,60,5402d0,5402f8,c8,40cc60
+ntoskrnl_17763-5122.exe,4d8bc0,4d87c0,4d89c0,6ca,409498,20,60,5402d0,5402f8,c8,40cc80
 ntoskrnl_18362-30.exe,500d60,500960,500b60,6fa,42fa40,20,50,56f390,56f3b8,c8,433200
 ntoskrnl_18362-116.exe,500de0,5009e0,500be0,6fa,42fa48,20,50,56f390,56f3b8,c8,433260
 ntoskrnl_18362-145.exe,500de0,5009e0,500be0,6fa,42f9e8,20,50,56f390,56f3b8,c8,433220
@@ -589,6 +592,7 @@ ntoskrnl_19041-3448.exe,cec460,cec260,cec060,87a,c19858,20,60,cfc410,cfc440,c8,c
 ntoskrnl_19041-3516.exe,cec1a0,cec5a0,cec3a0,87a,c197f8,20,60,cfc410,cfc440,c8,c1d900
 ntoskrnl_19041-3570.exe,cec660,cec460,cec260,87a,c197d8,20,60,cfc410,cfc440,c8,c1d900
 ntoskrnl_19041-3636.exe,cec5e0,cec3e0,cec1e0,87a,c197b8,20,60,cfc410,cfc440,c8,c1d8c0
+ntoskrnl_19041-3693.exe,cec120,cec520,cec320,87a,c19798,20,60,cfc410,cfc440,c8,c1d8e0
 ntoskrnl_22000-194.exe,cf5f40,cf5d40,cf6140,87a,c15d20,20,60,d06890,d068c0,c8,c1b7c0
 ntoskrnl_22000-258.exe,cf5f40,cf5d40,cf6140,87a,c15d20,20,60,d06890,d068c0,c8,c1b7c0
 ntoskrnl_22000-282.exe,cf5f00,cf5d00,cf6100,87a,c163d0,20,60,d06890,d068c0,c8,c1b7e0
@@ -666,3 +670,4 @@ ntoskrnl_22621-2283.exe,d0c440,d0c240,d0c040,87a,c318e0,20,60,d1da18,d1da40,c8,c
 ntoskrnl_22621-2361.exe,d0c510,d0c310,d0c110,87a,c318e0,20,60,d1da18,d1da40,c8,c374c0
 ntoskrnl_22621-2428.exe,d0c610,d0c410,d0c210,87a,c318e0,20,60,d1ea18,d1ea40,c8,c37560
 ntoskrnl_22621-2506.exe,d0c150,d0c550,d0c350,87a,c31880,20,60,d1ea18,d1ea40,c8,c37500
+ntoskrnl_22621-2715.exe,d0c150,d0c550,d0c350,87a,c31880,20,60,d1ea18,d1ea40,c8,c37500

Offsets/WdigestOffsets.csv

@@ -1,14 +1,16 @@
-imageVersion,g_fParameter_UseLogonCredential,g_IsCredGuardEnabled
+wdigestVersion,g_fParameter_UseLogonCredential,g_IsCredGuardEnabled
 wdigest_10240-16384.dll,35134,0
 wdigest_10240-17184.dll,35144,34ba0
 wdigest_10240-18244.dll,35144,34ba0
 wdigest_10240-18608.dll,35144,34ba0
 wdigest_10240-18638.dll,35144,34ba0
+wdigest_10240-20307.dll,35174,34ba0
 wdigest_10586-0.dll,35db0,35ba8
 wdigest_14393-0.dll,35dc0,35ba8
 wdigest_14393-3024.dll,35dc0,35ba8
 wdigest_14393-3750.dll,35dc0,35ba8
 wdigest_14393-3808.dll,35dc0,35ba8
+wdigest_14393-6451.dll,35de8,35ba8
 wdigest_15063-0.dll,34d8c,34b88
 wdigest_15063-1868.dll,34d8c,34b88
 wdigest_15063-2409.dll,34d8c,34b88
@@ -38,6 +40,7 @@ wdigest_17763-3887.dll,38234,37c08
 wdigest_17763-4011.dll,38234,37c08
 wdigest_17763-4131.dll,38234,37c08
 wdigest_17763-4974.dll,428c4,421b8
+wdigest_17763-5122.dll,428c4,421b8
 wdigest_18362-1.dll,35124,34b88
 wdigest_18362-175.dll,35124,34b88
 wdigest_18362-900.dll,35124,34b88
@@ -56,6 +59,8 @@ wdigest_19041-3505.dll,45a24,452e8
 wdigest_19041-3516.dll,45a14,452e8
 wdigest_19041-3570.dll,45a14,452e8
 wdigest_19041-3636.dll,45a14,452e8
+wdigest_19041-3684.dll,45a24,452e8
+wdigest_19041-3693.dll,45a24,452e8
 wdigest_22000-1.dll,3caa4,3cab0
 wdigest_22000-434.dll,3caa4,3cab0
 wdigest_22000-1030.dll,3caa4,3cab0
@@ -75,3 +80,5 @@ wdigest_22621-2070.dll,4b5ac,4b5b8
 wdigest_22621-2361.dll,4b59c,4b5a8
 wdigest_22621-2506.dll,4b59c,4b5a8
 wdigest_22621-2700.dll,4b59c,4b5a8
+wdigest_22621-2715.dll,4b5ac,4b5b8
+wdigest_22621-2771.dll,4b5ac,4b5b8