From eeefd835fe941c00615a1a20a62fae8591eecd1c Mon Sep 17 00:00:00 2001 From: Maxime Meignan Date: Wed, 29 Nov 2023 14:28:17 +0100 Subject: [PATCH] Refactored the extraction script for easier integration of new images/symbols --- Offsets/CiOffsets.csv | 1 + Offsets/ExtractOffsets.py | 49 +++++++++++++++++++++---------------- Offsets/NtoskrnlOffsets.csv | 7 +++++- Offsets/WdigestOffsets.csv | 9 ++++++- 4 files changed, 43 insertions(+), 23 deletions(-) diff --git a/Offsets/CiOffsets.csv b/Offsets/CiOffsets.csv index 33fc80d..92d9bb6 100644 --- a/Offsets/CiOffsets.csv +++ b/Offsets/CiOffsets.csv @@ -177,6 +177,7 @@ ci_17763-4644.dll,36d58,4bb30 ci_17763-4737.dll,36d58,4bb30 ci_17763-4840.dll,36d58,4bb30 ci_17763-4974.dll,36d58,4bb30 +ci_17763-5122.dll,36d58,4bb30 ci_17763-10458.dll,36d18,4ba70 ci_17763-10877.dll,36d18,4bae0 ci_18362-1.dll,37278,4c600 diff --git a/Offsets/ExtractOffsets.py b/Offsets/ExtractOffsets.py index c921808..7621fef 100644 --- a/Offsets/ExtractOffsets.py +++ b/Offsets/ExtractOffsets.py @@ -17,8 +17,12 @@ THREADS_LIMIT = None CSVLock = threading.Lock() machineType = dict(x86=332, x64=34404) -knownImageVersions = dict(ntoskrnl=list(), wdigest=list(), ci=list()) -extensions_by_mode = dict(ntoskrnl="exe", wdigest="dll", ci="dll") +supported_images = ["ntoskrnl.exe", "wdigest.dll", "ci.dll"] +modes = [image_name.split(".")[0] for image_name in supported_images] +extensions_by_mode = dict(image_name.split(".") for image_name in supported_images) +known_image_versions = {mode: list() for mode in modes} +modes_by_imagename = dict(zip(supported_images, modes)) +csvFilenameByMode = {mode: mode.capitalize() + "Offsets.csv" for mode in modes} symbols = dict( ntoskrnl=[ @@ -44,6 +48,8 @@ symbols = dict( ], ) +symbols_names = {mode: [t[0] if t[-1] == "symbol" else f"{t[0]}_{t[1]}" for t in symbols[mode]] for mode in modes} + def find(key: str, d: dict): for k, v in d.items(): @@ -252,12 +258,10 @@ def extractOffsets(input_file, output_file, mode): export_directory_rva = export_directory_entry.VirtualAddress image_name_rva = pe.get_dword_at_rva(export_directory_rva + 3 * 4) name = pe.get_string_at_rva(image_name_rva).decode().lower() - if "ntoskrnl.exe" in name: - imageType = "ntoskrnl" - elif "wdigest.dll" in name: - imageType = "wdigest" - elif "ci.dll" in name: - imageType = "ci" + for image_name in supported_images: + if image_name in name: + imageType = modes_by_imagename[image_name] + break else: print(f"[*] File {input_file} unrecognized") return @@ -274,7 +278,7 @@ def extractOffsets(input_file, output_file, mode): extension = extensions_by_mode[imageType] imageVersion = f"{imageType}_{full_version[2]}-{full_version[3]}.{extension}" - if imageVersion in knownImageVersions[imageType]: + if imageVersion in known_image_versions[imageType]: print(f"[*] Skipping known {imageType} version {imageVersion} (file: {input_file})") try: """ @@ -296,7 +300,7 @@ def extractOffsets(input_file, output_file, mode): for part in input_file_basename[len(f"{imageType}_") : -len(f".{extension}")].split("-") ) imageVersion = input_file_basename - if imageVersion in knownImageVersions[imageType]: + if imageVersion in known_image_versions[imageType]: return print("\r", end="") # Not skipping after all except ValueError: @@ -330,7 +334,7 @@ def extractOffsets(input_file, output_file, mode): # print("wrote into CSV !") del pdb - knownImageVersions[imageType].append(imageVersion) + known_image_versions[imageType].append(imageVersion) print(f"[+] Finished processing of {imageType} {input_file}!") except PEFormatError as e: @@ -381,22 +385,25 @@ def sortOutputFile(csvFile): if __name__ == "__main__": parser = argparse.ArgumentParser() + modes_str = "/".join(known_image_versions) + files = " / ".join(modes_by_imagename) + csvfiles = " / ".join(csvFilenameByMode.values()) parser.add_argument( "mode", - help='"ntoskrnl", "wdigest" or "ci". Mode to download and extract offsets from either ntoskrnl.exe, wdigest.dll or ci.dll', + help=f"{modes_str}. Mode to download and extract offsets from either {files}", ) parser.add_argument( "-i", "--input", dest="input", required=True, - help="Single file or directory containing ntoskrnl.exe / wdigest.dll / ci.dll to extract offsets from. If in download mode, the PE downloaded from MS symbols servers will be placed in this folder.", + help=f"Single file or directory containing {files} to extract offsets from. If in download mode, the PE downloaded from MS symbols servers will be placed in this folder.", ) parser.add_argument( "-o", "--output", dest="output", - help="CSV file to write offsets to. If the specified file already exists, only new ntoskrnl versions will be downloaded / analyzed. Defaults to NtoskrnlOffsets.csv / WdigestOffsets.csv / CiOffsets.csv in the current folder.", + help=f"CSV file to write offsets to. If the specified file already exists, only new ntoskrnl versions will be downloaded / analyzed. Defaults to {csvfiles} in the current folder.", ) parser.add_argument( "-d", @@ -408,20 +415,20 @@ if __name__ == "__main__": args = parser.parse_args() mode = args.mode.lower() - if mode not in knownImageVersions: - print(f'[!] ERROR : unsupported mode "{args.mode}", supported mode are: "ntoskrnl", "wdigest" and "ci"') + if mode not in known_image_versions: + print(f'[!] ERROR : unsupported mode "{args.mode}", supported mode are: {modes}') exit(1) # If the output file exists, load the already analyzed image versions. # Otherwise, write CSV headers to the new file. if not args.output: - args.output = mode.capitalize() + "Offsets.csv" + args.output = csvFilenameByMode[mode] if os.path.isfile(args.output): - loadOffsetsFromCSV(knownImageVersions[mode], args.output) - print(f'[+] Loaded {len(knownImageVersions[mode])} known {mode} versions from "{args.output}"') + loadOffsetsFromCSV(known_image_versions[mode], args.output) + print(f'[+] Loaded {len(known_image_versions[mode])} known {mode} versions from "{args.output}"') else: with open(args.output, "w") as output: - output.write(mode + "Version," + ",".join(elem[0] for elem in symbols[mode]) + "\n") + output.write(mode + "Version," + ",".join(elem for elem in symbols_names[mode]) + "\n") # In download mode, an updated list of image versions published will be retrieved from https://winbindex.m417z.com. # The symbols for each version will be downloaded from the Microsoft symbols servers. @@ -431,7 +438,7 @@ if __name__ == "__main__": print("[!] ERROR : in download mode, -i / --input option must specify a folder") exit(1) extension = extensions_by_mode[mode] - downloadPEFileFromMS(mode, extension, knownImageVersions[mode], args.input) + downloadPEFileFromMS(mode, extension, known_image_versions[mode], args.input) # Extract the offsets from the specified file or the folders containing image files. extractOffsets(args.input, args.output, mode) diff --git a/Offsets/NtoskrnlOffsets.csv b/Offsets/NtoskrnlOffsets.csv index acfc303..d387b53 100644 --- a/Offsets/NtoskrnlOffsets.csv +++ b/Offsets/NtoskrnlOffsets.csv @@ -1,4 +1,4 @@ -ntoskrnlVersion,PspCreateProcessNotifyRoutine,PspCreateThreadNotifyRoutine,PspLoadImageNotifyRoutine,_EPROCESS,EtwThreatIntProvRegHandle,_ETW_REG_ENTRY,_ETW_GUID_ENTRY,PsProcessType,PsThreadType,_OBJECT_TYPE,SeCiCallbacks +ntoskrnlVersion,PspCreateProcessNotifyRoutine,PspCreateThreadNotifyRoutine,PspLoadImageNotifyRoutine,_EPROCESS_Protection,EtwThreatIntProvRegHandle,_ETW_REG_ENTRY_GuidEntry,_ETW_GUID_ENTRY_ProviderEnableInfo,PsProcessType,PsThreadType,_OBJECT_TYPE_CallbackList,SeCiCallbacks ntoskrnl_10240-16384.exe,35d2e0,35d0e0,35cee0,6aa,0,20,50,3c51e8,3c5200,c8,31ee80 ntoskrnl_10240-17394.exe,35d420,35d220,35d020,6aa,0,20,50,3c51e8,3c5200,c8,31ef40 ntoskrnl_10240-17443.exe,35c420,35c220,35c020,6aa,0,20,50,3c41e8,3c4200,c8,31df40 @@ -55,6 +55,7 @@ ntoskrnl_10240-20048.exe,369520,369320,369120,6b2,0,20,50,3cf230,3cf248,c8,32b06 ntoskrnl_10240-20107.exe,3695a0,3693a0,3691a0,6b2,0,20,50,3cf228,3cf248,c8,32b0a0 ntoskrnl_10240-20161.exe,369560,369360,369160,6b2,0,20,50,3cf228,3cf248,c8,32b060 ntoskrnl_10240-20232.exe,369560,369360,369160,6b2,0,20,50,3cf228,3cf248,c8,32b060 +ntoskrnl_10240-20307.exe,369560,369360,369160,6b2,0,20,50,3cf228,3cf248,c8,32b060 ntoskrnl_10586-0.exe,317180,316f80,316d80,6b2,0,20,50,37f228,37f248,c8,2d8d40 ntoskrnl_10586-1176.exe,3161c0,315fc0,315dc0,6b2,0,20,50,37e228,37e248,c8,2d7d00 ntoskrnl_10586-1177.exe,3161c0,315fc0,315dc0,6b2,0,20,50,37e228,37e248,c8,2d7d00 @@ -154,6 +155,7 @@ ntoskrnl_14393-5921.exe,33ce20,33cc20,33ca20,6ca,0,20,50,3a9250,3a9278,c8,2fffa0 ntoskrnl_14393-5996.exe,33cf20,33cd20,33cb20,6ca,0,20,50,3a9250,3a9278,c8,300080 ntoskrnl_14393-6085.exe,33cea0,33cca0,33caa0,6ca,0,20,50,3a9250,3a9278,c8,300020 ntoskrnl_14393-6167.exe,33ce60,33cc60,33ca60,6ca,0,20,50,3a9250,3a9278,c8,300020 +ntoskrnl_14393-6451.exe,33cea0,33cca0,33caa0,6ca,0,20,50,3a9250,3a9278,c8,300040 ntoskrnl_15063-0.exe,382290,382090,381e90,6ca,341ea8,20,50,3e1f98,3e1fb0,c8,345be0 ntoskrnl_15063-13.exe,382290,382090,381e90,6ca,341ea8,20,50,3e1f98,3e1fb0,c8,345be0 ntoskrnl_15063-296.exe,382290,382090,381e90,6ca,341ea8,20,50,3e1f98,3e1fb0,c8,345be0 @@ -433,6 +435,7 @@ ntoskrnl_17763-4644.exe,4d8900,4d8b00,4d8700,6ca,409458,20,60,5402d0,5402f8,c8,4 ntoskrnl_17763-4737.exe,4d8940,4d8b40,4d8740,6ca,409478,20,60,5412d0,5412f8,c8,40cc40 ntoskrnl_17763-4851.exe,4d8c00,4d8800,4d8a00,6ca,4094b8,20,60,5412d0,5412f8,c8,40cca0 ntoskrnl_17763-4974.exe,4d8b40,4d8740,4d8940,6ca,409478,20,60,5402d0,5402f8,c8,40cc60 +ntoskrnl_17763-5122.exe,4d8bc0,4d87c0,4d89c0,6ca,409498,20,60,5402d0,5402f8,c8,40cc80 ntoskrnl_18362-30.exe,500d60,500960,500b60,6fa,42fa40,20,50,56f390,56f3b8,c8,433200 ntoskrnl_18362-116.exe,500de0,5009e0,500be0,6fa,42fa48,20,50,56f390,56f3b8,c8,433260 ntoskrnl_18362-145.exe,500de0,5009e0,500be0,6fa,42f9e8,20,50,56f390,56f3b8,c8,433220 @@ -589,6 +592,7 @@ ntoskrnl_19041-3448.exe,cec460,cec260,cec060,87a,c19858,20,60,cfc410,cfc440,c8,c ntoskrnl_19041-3516.exe,cec1a0,cec5a0,cec3a0,87a,c197f8,20,60,cfc410,cfc440,c8,c1d900 ntoskrnl_19041-3570.exe,cec660,cec460,cec260,87a,c197d8,20,60,cfc410,cfc440,c8,c1d900 ntoskrnl_19041-3636.exe,cec5e0,cec3e0,cec1e0,87a,c197b8,20,60,cfc410,cfc440,c8,c1d8c0 +ntoskrnl_19041-3693.exe,cec120,cec520,cec320,87a,c19798,20,60,cfc410,cfc440,c8,c1d8e0 ntoskrnl_22000-194.exe,cf5f40,cf5d40,cf6140,87a,c15d20,20,60,d06890,d068c0,c8,c1b7c0 ntoskrnl_22000-258.exe,cf5f40,cf5d40,cf6140,87a,c15d20,20,60,d06890,d068c0,c8,c1b7c0 ntoskrnl_22000-282.exe,cf5f00,cf5d00,cf6100,87a,c163d0,20,60,d06890,d068c0,c8,c1b7e0 @@ -666,3 +670,4 @@ ntoskrnl_22621-2283.exe,d0c440,d0c240,d0c040,87a,c318e0,20,60,d1da18,d1da40,c8,c ntoskrnl_22621-2361.exe,d0c510,d0c310,d0c110,87a,c318e0,20,60,d1da18,d1da40,c8,c374c0 ntoskrnl_22621-2428.exe,d0c610,d0c410,d0c210,87a,c318e0,20,60,d1ea18,d1ea40,c8,c37560 ntoskrnl_22621-2506.exe,d0c150,d0c550,d0c350,87a,c31880,20,60,d1ea18,d1ea40,c8,c37500 +ntoskrnl_22621-2715.exe,d0c150,d0c550,d0c350,87a,c31880,20,60,d1ea18,d1ea40,c8,c37500 diff --git a/Offsets/WdigestOffsets.csv b/Offsets/WdigestOffsets.csv index eeb8fd2..a6f5fe4 100644 --- a/Offsets/WdigestOffsets.csv +++ b/Offsets/WdigestOffsets.csv @@ -1,14 +1,16 @@ -imageVersion,g_fParameter_UseLogonCredential,g_IsCredGuardEnabled +wdigestVersion,g_fParameter_UseLogonCredential,g_IsCredGuardEnabled wdigest_10240-16384.dll,35134,0 wdigest_10240-17184.dll,35144,34ba0 wdigest_10240-18244.dll,35144,34ba0 wdigest_10240-18608.dll,35144,34ba0 wdigest_10240-18638.dll,35144,34ba0 +wdigest_10240-20307.dll,35174,34ba0 wdigest_10586-0.dll,35db0,35ba8 wdigest_14393-0.dll,35dc0,35ba8 wdigest_14393-3024.dll,35dc0,35ba8 wdigest_14393-3750.dll,35dc0,35ba8 wdigest_14393-3808.dll,35dc0,35ba8 +wdigest_14393-6451.dll,35de8,35ba8 wdigest_15063-0.dll,34d8c,34b88 wdigest_15063-1868.dll,34d8c,34b88 wdigest_15063-2409.dll,34d8c,34b88 @@ -38,6 +40,7 @@ wdigest_17763-3887.dll,38234,37c08 wdigest_17763-4011.dll,38234,37c08 wdigest_17763-4131.dll,38234,37c08 wdigest_17763-4974.dll,428c4,421b8 +wdigest_17763-5122.dll,428c4,421b8 wdigest_18362-1.dll,35124,34b88 wdigest_18362-175.dll,35124,34b88 wdigest_18362-900.dll,35124,34b88 @@ -56,6 +59,8 @@ wdigest_19041-3505.dll,45a24,452e8 wdigest_19041-3516.dll,45a14,452e8 wdigest_19041-3570.dll,45a14,452e8 wdigest_19041-3636.dll,45a14,452e8 +wdigest_19041-3684.dll,45a24,452e8 +wdigest_19041-3693.dll,45a24,452e8 wdigest_22000-1.dll,3caa4,3cab0 wdigest_22000-434.dll,3caa4,3cab0 wdigest_22000-1030.dll,3caa4,3cab0 @@ -75,3 +80,5 @@ wdigest_22621-2070.dll,4b5ac,4b5b8 wdigest_22621-2361.dll,4b59c,4b5a8 wdigest_22621-2506.dll,4b59c,4b5a8 wdigest_22621-2700.dll,4b59c,4b5a8 +wdigest_22621-2715.dll,4b5ac,4b5b8 +wdigest_22621-2771.dll,4b5ac,4b5b8