Return to Answer

deleted 67 characters in body

edited Apr 7, 2020 at 20:52

import sys #litte-endian integer readers def read_leuint8(file): data = file.read(1) return data[0] def read_leuint16(file): data = file.read(2) return data[0] | (data[1] << 8) def read_leuint24(file): data = file.read(3) return data[0] | (data[1] << 8) | (data[2] << 16) def read_leuint32(file): data = file.read(4) return data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24) class MEFileSystemFileMetadataStateMachine: """ MEFileSystemFileMetadata: Files in MFS have internal metadata entries. Each file begins with a metadata record, observed values are: 0xa# <bytes ahead until next metadata> <0x01> <next meta block num> <0x00> 0xb# <blocks ahead until next metadata> <0x01> <next meta block num> <0x00> 0x8# <bytes remaining in file (including this metadata)> where # -- is the file number within the block. This is the target for the fno field of the allocation table entry. <bytes ahead...> -- the number of bytes to move ahead to find the next metadata record (including this metadata). <next meta block...>-- the 0x100-byte block number where the next metadata record is to be found. This value should be looked up through the data-page's block-indirection table. <blocks ahead...> -- the number of 0x100 byte blocks to move ahead to find the next metadata record (including this metadata). <bytes remaining...>-- pretty straight-forward. So, 0x8# metadata are file-end records; 0xa# are short-range references; and 0xb# are longer range references. This metadata chain provides unamiguous file numbers within the blocks, and since they're put at the block start of any block that contains a file-start, it's easy to pick up from an allocation table reference. Note: 0x8# records don't point to the next metadata block, so we may have to consume file-end padding (0xff until the next multiple of 0x10) if we get an intermediate 0x8# metadata while searching for our target file. """ STATE_NEED_META = 0 STATE_NEED_SKIP_DATA = 1 STATE_NEED_FILE_DATA = 2 STATE_COMPLETE = 3 def __init__(self, file_no, file_len): self.file_no = file_no self.state = MEFileSystemFileMetadataStateMachine.STATE_NEED_META self.bytes_needed = 1 self.byte_offset = 0 self.cur_meta = bytearray(5) self.file_data = bytearray(file_len) self.file_filled = 0 self.found_fileno = False self.work_buf = self.cur_meta def is_complete(self): return self.state == self.STATE_COMPLETE def get_file_data(self): return self.file_data def get_bytes_needed(self): return self.bytes_needed #returns the number of bytes consumed def add_bytes(self, bytes, start_index, data_len=None, log_file = None): """ supplies data to satisfy the state-machine's need for data as reported via get_bytes_needed(). bytes -- the buffer containing the bytes to be fed in start_index -- the start location of the bytes within the buffer data_len -- number of bytes in the array, starting at start_index. if None, then len(bytes) - start_index is assumed """ #shuffling data from potentially multiple calls to fill the data request from the #state machine (get_bytes_needed) data_len = len(bytes) - start_index if data_len is None else data_len if data_len == 0: return 0 # nothing to do #take the min of what's available and what we need to_copy = data_len if data_len < self.bytes_needed else self.bytes_needed if self.work_buf: self.work_buf[self.byte_offset:(self.byte_offset+to_copy)] = bytes[start_index:(start_index+to_copy)] self.byte_offset = self.byte_offset + to_copy self.bytes_needed = self.bytes_needed - to_copy #if we don't have enough to process, return so they can feed more if self.bytes_needed > 0: return to_copy #we only make it this far once we've got the full bytes_needed data meta_type = self.cur_meta[0] & 0xf0 if self.state == self.STATE_NEED_META: if self.byte_offset == 1: if meta_type in [0xa0, 0xb0]: self.bytes_needed = 4 else: self.bytes_needed = 1 else: #Have we found the file number we seek yet? if self.found_fileno or (self.file_no == self.cur_meta[0] & 0x0f): self.found_fileno = True self.state = self.STATE_NEED_FILE_DATA self.work_buf = self.file_data self.byte_offset = self.file_filled else: self.state = self.STATE_NEED_SKIP_DATA self.work_buf = None self.byte_offset = None #determine the data required based on metadata type, and whether we're #skipping (so need to eat EOF padding on type 0x8# entries) or whether #we're copying out file data. if meta_type == 0x80: if self.state == self.STATE_NEED_SKIP_DATA: #if we're skipping a 0x8# entry, we need to eat EOF padding too padding = (0x10 - (self.cur_meta[1] & 0xf)) & 0xf self.bytes_needed = padding + self.cur_meta[1] - 2 #remove header len, too else: self.bytes_needed = self.cur_meta[1] - 2 #remove header len elif meta_type == 0xa0: self.bytes_needed = self.cur_meta[1] - 5 #remove header len elif meta_type == 0xb0: self.bytes_needed = self.cur_meta[1] * 0x100 - 5 #remove header len else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_NEED_SKIP_DATA: #recall: this is the state just *completed* self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 self.bytes_needed = 1 elif self.state == self.STATE_NEED_FILE_DATA: #recall: this is the state just *completed* self.file_filled = self.byte_offset self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 if meta_type == 0x80: #just completed a file-end record...we're done. self.bytes_needed = 0 self.state = self.STATE_COMPLETE elif meta_type in [0xa0, 0xb0]: self.bytes_needed = 1 else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_COMPLETE: #can't leave this state pass else: if log_file: log_file.write("Bad state-machine state: %d\n" % self.state) #recurse to consume as much as we can, for easier calling convention if to_copy < data_len: return to_copy + add_bytes(bytes, start_index+to_copy, data_len-to_copy) #else, return what we consumed return to_copy def read_me_fs_file(file_no, file_len, me_file, log_file = sys.stdout): sm = MEFileSystemFileMetadataStateMachine(file_no, file_len) while not sm.is_complete(): res = sm.add_bytes( bytes=me_file.read(sm.get_bytes_needed()), start_index=0, data_len=None, #shorthand for len(bytes)-start_index log_file=log_file) if not res: log_file.write("Aborting file read.\n") break return sm.get_file_data() class MEFileSystemDataHeader: """ Data Page Header: (Top of a 0x4000-byte data page in the MFS) 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| pgflags | 0x00| 0x00| 0x00| 0x00| mystery bits | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | freed_flags... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ 80 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 90 | block indirection table... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ c0 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ d0+| (file data follows) | +-----------------------------------------------------------------------------------------------+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. pgflags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. myst_bits -- TBD freed_flags-- each bit marks whether a written 0x10-byte chunk is no longer required. Bits exist for the header chunks, as well as the chunks for file data. lsb of the first byte corresponds to the first chunk of the header. 0=freed. blk_itab -- translates ATFileEntry's okey value into a block offset within the data page. offset = blk_itab[okey] * 0x100. This is the location from which to begin the search. Note that an offset of 0 must still respect that the page header consumes bytes [0x00, 0xd0) for the search. (filedata) -- File are padded with 0xff at the end. Note files seem to have at least 2 bytes more data than  the specifiedinternal lengthmetadata.  Larger filesSee haveroutines moreabove. """ def __init__(self, page_no, flags, bytes_00000000, myst_bits, freed_flags, blk_itab): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.myst_bits = myst_bits self.freed_flags = freed_flags self.blk_itab = blk_itab def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemDataHeader %x:\n" % id(self)) if self.page_no != 0xff: log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" mystery bits: %s\n" % " ".join("%02x"%x for x in self.myst_bits)) log.write(" freed_flags: %s\n" % "".join("%02x"%x for x in self.freed_flags)) log.write(" block ind. tab.: [%s]\n" % " ".join("%02x"%x for x in self.blk_itab)) else: log.write(" (empty)\n") class MEFileSystemATEntry: """ Allocation Table Entry: 0 1 2 3 4 5 6 7 8 9 a +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 |state| flgs| identifier | type| filelen | pgid| okey| fno | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ state -- status of the entry: 0xdc=present; 0xc8=overwritten(i.e., there will be another entry below) flags -- has value 0xf0 in every example...so hard to tell. identifier -- 3-byte identifier...seem to be a preference for ASCII, but there are counterexamples type -- no idea...maybe permissions? Observed values: 0x00, 0x0a, 0x0c, 0x0d, 0x0e, 0x18, 0x1a filelen -- 16-bit, little endian. actual content seems to be a few bytes more...might just be pollution of the structures used to write the file. pgid -- the pgno for the MEFileSystemDataHeader that holds the data. The ATHeader is numbered 0, each data page seems to get sequentially numbered in my examples, though that could change with more use. 0xff indicates "not yet used / nonumber assigned" --- a fact that hints page numbers aren't guaranteed to be sequentially okey -- key for indexing the block_indirection table of the MEFileSystemDataHeader holding this file's content, to find the right 0x100 byte block from which to start the file search according to the 'fno' field. fno -- file number within the block-offset determined from okey. This is to be looked up using the metadata *within* the file data of the data pages. See MEFileSystemFileMetadataStateMachine above """ def __init__(self, state, flags, identifier, type, filelen, pgid, okey, fno): self.state = state self.flags = flags self.identifier = identifier self.type = type self.filelen = filelen self.pgid = pgid self.okey = okey self.fno = fno def debug_print(self, log = sys.stdout): log.write("%15s len=0x%04x [pg=0x%02x k=0x%02x f=0x%02x] ty=0x%02x st=0x%02x, fg=0x%02x" % (str(self.identifier), self.filelen, self.pgid, self.okey, self.fno, self.type, self.state, self.flags)) class MEFileSystemATHeader: """ Allocation Table Header: 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| flags | 0x00| 0x00| 0x00| 0x00|"MFS\0" |bitfields? | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | bitfields? | +-----+-----+-----+-----+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. flags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. "MFS\0" -- ASCIIZ signature for the MFS AT Header bitfields -- 64 bits of apparent bitfields """ max_files = int((0x4000 - 0x14) / 11) #page size, less the header, divided by file entry size def __init__(self, page_no, flags, bytes_00000000, sig, bitfields): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.sig_good = sig == b'MFS\x00' self.bitfields = bitfields def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemATHeader %x:\n" % id(self)) log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" sig good: %s\n" % str(self.sig_good)) log.write(" bitfields: %s\n" % " ".join("%02x"%x for x in self.bitfields)) class MEFileSystemAT: """ Allocation Table (a 0x4000-byte page in the MFS): 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | Allocation Table Header... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 |... | File Entry |FE...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ ...|... | +-----------------------------------------------------------------------------------+-----+-----+ 3fe0|... |File Ent...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 3ff0|... |Padding | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ """ def __init__(self, header, entry_list): self.header = header self.entry_list = entry_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemAT %x:\n" % id(self)) self.header.debug_print(log) log.write("File entries:\n") for ent in self.entry_list: if(ent): log.write(" ") ent.debug_print(log) log.write("\n") class MEFileSystem: """ MFS Partition: (Making the assumption the AT is the first page) +------------------------------------+ 0000 | Allocation Table | +------------------------------------+ 4000 | Data Page | +------------------------------------+ 8000 | Data Page | +------------------------------------+ c000 | Data Page | +------------------------------------+ ... |... | +------------------------------------+ """ def __init__(self, allocation_table, data_page_list): self.allocation_table = allocation_table self.data_page_list = data_page_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystem %x:\n" % id(self)) self.allocation_table.debug_print(log) for page in self.data_page_list: if(page): page.debug_print(log) else: log.write("(Not parsed)\n") def parse_me_fs_at_entry(me_file, log_file = sys.stdout): return MEFileSystemATEntry( state = read_leuint8(me_file), flags = read_leuint8(me_file), identifier = me_file.read(3), type = read_leuint8(me_file), filelen = read_leuint16(me_file), pgid = read_leuint8(me_file), okey = read_leuint8(me_file), fno = read_leuint8(me_file)) def parse_me_fs_at_header(me_file, log_file = sys.stdout): return MEFileSystemATHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), sig = me_file.read(4), bitfields = me_file.read(8)) def parse_me_fs_at(me_file, log_file = sys.stdout): hdr = parse_me_fs_at_header(me_file, log_file) entry_list = [None] * MEFileSystemATHeader.max_files for i in range(MEFileSystemATHeader.max_files): ent = parse_me_fs_at_entry(me_file, log_file) if ent.state == 0xff: break entry_list[i] = ent return MEFileSystemAT(hdr, entry_list) def parse_me_fs_data_header(me_file, log_file = sys.stdout): return MEFileSystemDataHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), myst_bits = me_file.read(0x8), freed_flags = me_file.read(0x80), blk_itab = me_file.read(0x40)) def parse_me_fs(length, me_file, log_file = sys.stdout): """ ARGS: length -- length in bytes of the ME partition holding the MFS/MFSB data me_file -- a file handle whose present position is the start of the MFS(B) partition log_file -- if there is diagnostic output, put it here RETURNS: an MEFileSystem instance populated with the allocation table and data-page headers """ total_pages = int(length/0x4000) start_offset = me_file.tell() #NOTE: I'm presuming the allocation table is the first page... #that might not be reliable... at = parse_me_fs_at(me_file, log_file) data_pages = [None] * (total_pages-1) for page in range(0, total_pages-1): me_file.seek(start_offset + 0x4000 * (page+1)) data_pages[page] = parse_me_fs_data_header(me_file, log_file) return MEFileSystem( allocation_table=at, data_page_list = data_pages) def get_mfs_file(mefs, me_file, mfs_file_offset, id, log_file = sys.stdout): """ Example of how to use the MEFileSystem structures to retrieve MFS and MFSB files. ARGS: mefs -- a MEFileSystem instance parsed from mfs_data me_file -- a filehandle containing the ME image mfs_file_offset -- the file offset within me_file where the MFS partition begins id -- a 3-byte byte array with the file identifier log -- if there is diagnostic output, put it here RETURNS: an array containing [state, The data from the corresponding file]. else None if the file identifier does not exist within the data. Example driver, given the known offset and size for a MFS partition: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) result_tuple = get_mfs_file(mefs, spi_image_file, MFS_OFFSET, b'UKS') if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple)) """ #Find the file identifer in the Allocation Table best_ent = None for ent in mefs.allocation_table.entry_list: if ent and ent.identifier == id: best_ent = ent if ent.state == 0xdc: break; # got a current entry log.write("Error: found an item w/ state %02x...continuing\n" % ent.state) #if found, lookup which data page matches the entry's pgid value if best_ent: page_found = False for list_idx in range(len(mefs.data_page_list)): page = mefs.data_page_list[list_idx] if page.page_no == best_ent.pgid: page_found = True #we found the right data page, so start the file search search_start = page.blk_itab[best_ent.okey] * 0x100 #In the following lines: # The value d0 is to skip over the datapage header if we're in the first block # # The multiple of 0x4000 selects the data offset that goes with list_idx # since the parsed data-page list is in the same order as found in the file. # # Because mefs.data_page_list doesn't include the allocation table page, we +1 # to the index before multiplying. The result is a set of offsets into the MFS data # bounding the file search ## search_off = 0x4000 * (list_idx+1) + (0xd0 if search_start == 0 else search_start) me_file.seek(mfs_file_offset + search_off) data = read_me_fs_file(best_ent.fno, best_ent.filelen, me_file, log_file) if data: return [best_ent.state, data] return None if __name__ == "__main__": with open("image.bin", "rb") as spi_image_file: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) #Dump the allocation table mefs.allocation_table.debug_print() print("") first_file = mefs.allocation_table.entry_list[0].identifier print("looking up the first file (%s):" % first_file) result_tuple = get_mfs_file(mefs, spi_image_file, MFS_OFFSET, first_file) if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple))

import sys #litte-endian integer readers def read_leuint8(file): data = file.read(1) return data[0] def read_leuint16(file): data = file.read(2) return data[0] | (data[1] << 8) def read_leuint24(file): data = file.read(3) return data[0] | (data[1] << 8) | (data[2] << 16) def read_leuint32(file): data = file.read(4) return data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24) class MEFileSystemFileMetadataStateMachine: """ MEFileSystemFileMetadata: Files in MFS have internal metadata entries. Each file begins with a metadata record, observed values are: 0xa# <bytes ahead until next metadata> <0x01> <next meta block num> <0x00> 0xb# <blocks ahead until next metadata> <0x01> <next meta block num> <0x00> 0x8# <bytes remaining in file (including this metadata)> where # -- is the file number within the block. This is the target for the fno field of the allocation table entry. <bytes ahead...> -- the number of bytes to move ahead to find the next metadata record (including this metadata). <next meta block...>-- the 0x100-byte block number where the next metadata record is to be found. This value should be looked up through the data-page's block-indirection table. <blocks ahead...> -- the number of 0x100 byte blocks to move ahead to find the next metadata record (including this metadata). <bytes remaining...>-- pretty straight-forward. So, 0x8# metadata are file-end records; 0xa# are short-range references; and 0xb# are longer range references. This metadata chain provides unamiguous file numbers within the blocks, and since they're put at the block start of any block that contains a file-start, it's easy to pick up from an allocation table reference. Note: 0x8# records don't point to the next metadata block, so we may have to consume file-end padding (0xff until the next multiple of 0x10) if we get an intermediate 0x8# metadata while searching for our target file. """ STATE_NEED_META = 0 STATE_NEED_SKIP_DATA = 1 STATE_NEED_FILE_DATA = 2 STATE_COMPLETE = 3 def __init__(self, file_no, file_len): self.file_no = file_no self.state = MEFileSystemFileMetadataStateMachine.STATE_NEED_META self.bytes_needed = 1 self.byte_offset = 0 self.cur_meta = bytearray(5) self.file_data = bytearray(file_len) self.file_filled = 0 self.found_fileno = False self.work_buf = self.cur_meta def is_complete(self): return self.state == self.STATE_COMPLETE def get_file_data(self): return self.file_data def get_bytes_needed(self): return self.bytes_needed #returns the number of bytes consumed def add_bytes(self, bytes, start_index, data_len=None, log_file = None): """ supplies data to satisfy the state-machine's need for data as reported via get_bytes_needed(). bytes -- the buffer containing the bytes to be fed in start_index -- the start location of the bytes within the buffer data_len -- number of bytes in the array, starting at start_index. if None, then len(bytes) - start_index is assumed """ #shuffling data from potentially multiple calls to fill the data request from the #state machine (get_bytes_needed) data_len = len(bytes) - start_index if data_len is None else data_len if data_len == 0: return 0 # nothing to do #take the min of what's available and what we need to_copy = data_len if data_len < self.bytes_needed else self.bytes_needed if self.work_buf: self.work_buf[self.byte_offset:(self.byte_offset+to_copy)] = bytes[start_index:(start_index+to_copy)] self.byte_offset = self.byte_offset + to_copy self.bytes_needed = self.bytes_needed - to_copy #if we don't have enough to process, return so they can feed more if self.bytes_needed > 0: return to_copy #we only make it this far once we've got the full bytes_needed data meta_type = self.cur_meta[0] & 0xf0 if self.state == self.STATE_NEED_META: if self.byte_offset == 1: if meta_type in [0xa0, 0xb0]: self.bytes_needed = 4 else: self.bytes_needed = 1 else: #Have we found the file number we seek yet? if self.found_fileno or (self.file_no == self.cur_meta[0] & 0x0f): self.found_fileno = True self.state = self.STATE_NEED_FILE_DATA self.work_buf = self.file_data self.byte_offset = self.file_filled else: self.state = self.STATE_NEED_SKIP_DATA self.work_buf = None self.byte_offset = None #determine the data required based on metadata type, and whether we're #skipping (so need to eat EOF padding on type 0x8# entries) or whether #we're copying out file data. if meta_type == 0x80: if self.state == self.STATE_NEED_SKIP_DATA: #if we're skipping a 0x8# entry, we need to eat EOF padding too padding = (0x10 - (self.cur_meta[1] & 0xf)) & 0xf self.bytes_needed = padding + self.cur_meta[1] - 2 #remove header len, too else: self.bytes_needed = self.cur_meta[1] - 2 #remove header len elif meta_type == 0xa0: self.bytes_needed = self.cur_meta[1] - 5 #remove header len elif meta_type == 0xb0: self.bytes_needed = self.cur_meta[1] * 0x100 - 5 #remove header len else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_NEED_SKIP_DATA: #recall: this is the state just *completed* self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 self.bytes_needed = 1 elif self.state == self.STATE_NEED_FILE_DATA: #recall: this is the state just *completed* self.file_filled = self.byte_offset self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 if meta_type == 0x80: #just completed a file-end record...we're done. self.bytes_needed = 0 self.state = self.STATE_COMPLETE elif meta_type in [0xa0, 0xb0]: self.bytes_needed = 1 else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_COMPLETE: #can't leave this state pass else: if log_file: log_file.write("Bad state-machine state: %d\n" % self.state) #recurse to consume as much as we can, for easier calling convention if to_copy < data_len: return to_copy + add_bytes(bytes, start_index+to_copy, data_len-to_copy) #else, return what we consumed return to_copy def read_me_fs_file(file_no, file_len, me_file, log_file = sys.stdout): sm = MEFileSystemFileMetadataStateMachine(file_no, file_len) while not sm.is_complete(): res = sm.add_bytes( bytes=me_file.read(sm.get_bytes_needed()), start_index=0, data_len=None, #shorthand for len(bytes)-start_index log_file=log_file) if not res: log_file.write("Aborting file read.\n") break return sm.get_file_data() class MEFileSystemDataHeader: """ Data Page Header: (Top of a 0x4000-byte data page in the MFS) 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| pgflags | 0x00| 0x00| 0x00| 0x00| mystery bits | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | freed_flags... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ 80 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 90 | block indirection table... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ c0 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ d0+| (file data follows) | +-----------------------------------------------------------------------------------------------+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. pgflags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. myst_bits -- TBD freed_flags-- each bit marks whether a written 0x10-byte chunk is no longer required. Bits exist for the header chunks, as well as the chunks for file data. lsb of the first byte corresponds to the first chunk of the header. 0=freed. blk_itab -- translates ATFileEntry's okey value into a block offset within the data page. offset = blk_itab[okey] * 0x100. This is the location from which to begin the search. Note that an offset of 0 must still respect that the page header consumes bytes [0x00, 0xd0) for the search. (filedata) -- File are padded with 0xff at the end. Note files seem to have at least 2 bytes more data than  the specified length.  Larger files have more. """ def __init__(self, page_no, flags, bytes_00000000, myst_bits, freed_flags, blk_itab): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.myst_bits = myst_bits self.freed_flags = freed_flags self.blk_itab = blk_itab def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemDataHeader %x:\n" % id(self)) if self.page_no != 0xff: log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" mystery bits: %s\n" % " ".join("%02x"%x for x in self.myst_bits)) log.write(" freed_flags: %s\n" % "".join("%02x"%x for x in self.freed_flags)) log.write(" block ind. tab.: [%s]\n" % " ".join("%02x"%x for x in self.blk_itab)) else: log.write(" (empty)\n") class MEFileSystemATEntry: """ Allocation Table Entry: 0 1 2 3 4 5 6 7 8 9 a +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 |state| flgs| identifier | type| filelen | pgid| okey| fno | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ state -- status of the entry: 0xdc=present; 0xc8=overwritten(i.e., there will be another entry below) flags -- has value 0xf0 in every example...so hard to tell. identifier -- 3-byte identifier...seem to be a preference for ASCII, but there are counterexamples type -- no idea...maybe permissions? Observed values: 0x00, 0x0a, 0x0c, 0x0d, 0x0e, 0x18, 0x1a filelen -- 16-bit, little endian. actual content seems to be a few bytes more...might just be pollution of the structures used to write the file. pgid -- the pgno for the MEFileSystemDataHeader that holds the data. The ATHeader is numbered 0, each data page seems to get sequentially numbered in my examples, though that could change with more use. 0xff indicates "not yet used / nonumber assigned" --- a fact that hints page numbers aren't guaranteed to be sequentially okey -- key for indexing the block_indirection table of the MEFileSystemDataHeader holding this file's content, to find the right 0x100 byte block from which to start the file search according to the 'fno' field. fno -- file number within the block-offset determined from okey. This is to be looked up using the metadata *within* the file data of the data pages. See MEFileSystemFileMetadataStateMachine above """ def __init__(self, state, flags, identifier, type, filelen, pgid, okey, fno): self.state = state self.flags = flags self.identifier = identifier self.type = type self.filelen = filelen self.pgid = pgid self.okey = okey self.fno = fno def debug_print(self, log = sys.stdout): log.write("%15s len=0x%04x [pg=0x%02x k=0x%02x f=0x%02x] ty=0x%02x st=0x%02x, fg=0x%02x" % (str(self.identifier), self.filelen, self.pgid, self.okey, self.fno, self.type, self.state, self.flags)) class MEFileSystemATHeader: """ Allocation Table Header: 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| flags | 0x00| 0x00| 0x00| 0x00|"MFS\0" |bitfields? | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | bitfields? | +-----+-----+-----+-----+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. flags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. "MFS\0" -- ASCIIZ signature for the MFS AT Header bitfields -- 64 bits of apparent bitfields """ max_files = int((0x4000 - 0x14) / 11) #page size, less the header, divided by file entry size def __init__(self, page_no, flags, bytes_00000000, sig, bitfields): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.sig_good = sig == b'MFS\x00' self.bitfields = bitfields def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemATHeader %x:\n" % id(self)) log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" sig good: %s\n" % str(self.sig_good)) log.write(" bitfields: %s\n" % " ".join("%02x"%x for x in self.bitfields)) class MEFileSystemAT: """ Allocation Table (a 0x4000-byte page in the MFS): 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | Allocation Table Header... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 |... | File Entry |FE...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ ...|... | +-----------------------------------------------------------------------------------+-----+-----+ 3fe0|... |File Ent...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 3ff0|... |Padding | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ """ def __init__(self, header, entry_list): self.header = header self.entry_list = entry_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemAT %x:\n" % id(self)) self.header.debug_print(log) log.write("File entries:\n") for ent in self.entry_list: if(ent): log.write(" ") ent.debug_print(log) log.write("\n") class MEFileSystem: """ MFS Partition: (Making the assumption the AT is the first page) +------------------------------------+ 0000 | Allocation Table | +------------------------------------+ 4000 | Data Page | +------------------------------------+ 8000 | Data Page | +------------------------------------+ c000 | Data Page | +------------------------------------+ ... |... | +------------------------------------+ """ def __init__(self, allocation_table, data_page_list): self.allocation_table = allocation_table self.data_page_list = data_page_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystem %x:\n" % id(self)) self.allocation_table.debug_print(log) for page in self.data_page_list: if(page): page.debug_print(log) else: log.write("(Not parsed)\n") def parse_me_fs_at_entry(me_file, log_file = sys.stdout): return MEFileSystemATEntry( state = read_leuint8(me_file), flags = read_leuint8(me_file), identifier = me_file.read(3), type = read_leuint8(me_file), filelen = read_leuint16(me_file), pgid = read_leuint8(me_file), okey = read_leuint8(me_file), fno = read_leuint8(me_file)) def parse_me_fs_at_header(me_file, log_file = sys.stdout): return MEFileSystemATHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), sig = me_file.read(4), bitfields = me_file.read(8)) def parse_me_fs_at(me_file, log_file = sys.stdout): hdr = parse_me_fs_at_header(me_file, log_file) entry_list = [None] * MEFileSystemATHeader.max_files for i in range(MEFileSystemATHeader.max_files): ent = parse_me_fs_at_entry(me_file, log_file) if ent.state == 0xff: break entry_list[i] = ent return MEFileSystemAT(hdr, entry_list) def parse_me_fs_data_header(me_file, log_file = sys.stdout): return MEFileSystemDataHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), myst_bits = me_file.read(0x8), freed_flags = me_file.read(0x80), blk_itab = me_file.read(0x40)) def parse_me_fs(length, me_file, log_file = sys.stdout): """ ARGS: length -- length in bytes of the ME partition holding the MFS/MFSB data me_file -- a file handle whose present position is the start of the MFS(B) partition log_file -- if there is diagnostic output, put it here RETURNS: an MEFileSystem instance populated with the allocation table and data-page headers """ total_pages = int(length/0x4000) start_offset = me_file.tell() #NOTE: I'm presuming the allocation table is the first page... #that might not be reliable... at = parse_me_fs_at(me_file, log_file) data_pages = [None] * (total_pages-1) for page in range(0, total_pages-1): me_file.seek(start_offset + 0x4000 * (page+1)) data_pages[page] = parse_me_fs_data_header(me_file, log_file) return MEFileSystem( allocation_table=at, data_page_list = data_pages) def get_mfs_file(mefs, me_file, mfs_file_offset, id, log_file = sys.stdout): """ Example of how to use the MEFileSystem structures to retrieve MFS and MFSB files. ARGS: mefs -- a MEFileSystem instance parsed from mfs_data me_file -- a filehandle containing the ME image mfs_file_offset -- the file offset within me_file where the MFS partition begins id -- a 3-byte byte array with the file identifier log -- if there is diagnostic output, put it here RETURNS: an array containing [state, The data from the corresponding file]. else None if the file identifier does not exist within the data. Example driver, given the known offset and size for a MFS partition: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) result_tuple = get_mfs_file(mefs, spi_image_file, MFS_OFFSET, b'UKS') if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple)) """ #Find the file identifer in the Allocation Table best_ent = None for ent in mefs.allocation_table.entry_list: if ent and ent.identifier == id: best_ent = ent if ent.state == 0xdc: break; # got a current entry log.write("Error: found an item w/ state %02x...continuing\n" % ent.state) #if found, lookup which data page matches the entry's pgid value if best_ent: page_found = False for list_idx in range(len(mefs.data_page_list)): page = mefs.data_page_list[list_idx] if page.page_no == best_ent.pgid: page_found = True #we found the right data page, so start the file search search_start = page.blk_itab[best_ent.okey] * 0x100 #In the following lines: # The value d0 is to skip over the datapage header if we're in the first block # # The multiple of 0x4000 selects the data offset that goes with list_idx # since the parsed data-page list is in the same order as found in the file. # # Because mefs.data_page_list doesn't include the allocation table page, we +1 # to the index before multiplying. The result is a set of offsets into the MFS data # bounding the file search ## search_off = 0x4000 * (list_idx+1) + (0xd0 if search_start == 0 else search_start) me_file.seek(mfs_file_offset + search_off) data = read_me_fs_file(best_ent.fno, best_ent.filelen, me_file, log_file) if data: return [best_ent.state, data] return None if __name__ == "__main__": with open("image.bin", "rb") as spi_image_file: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) #Dump the allocation table mefs.allocation_table.debug_print() print("") first_file = mefs.allocation_table.entry_list[0].identifier print("looking up the first file (%s):" % first_file) result_tuple = get_mfs_file(mefs, spi_image_file, MFS_OFFSET, first_file) if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple))

import sys #litte-endian integer readers def read_leuint8(file): data = file.read(1) return data[0] def read_leuint16(file): data = file.read(2) return data[0] | (data[1] << 8) def read_leuint24(file): data = file.read(3) return data[0] | (data[1] << 8) | (data[2] << 16) def read_leuint32(file): data = file.read(4) return data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24) class MEFileSystemFileMetadataStateMachine: """ MEFileSystemFileMetadata: Files in MFS have internal metadata entries. Each file begins with a metadata record, observed values are: 0xa# <bytes ahead until next metadata> <0x01> <next meta block num> <0x00> 0xb# <blocks ahead until next metadata> <0x01> <next meta block num> <0x00> 0x8# <bytes remaining in file (including this metadata)> where # -- is the file number within the block. This is the target for the fno field of the allocation table entry. <bytes ahead...> -- the number of bytes to move ahead to find the next metadata record (including this metadata). <next meta block...>-- the 0x100-byte block number where the next metadata record is to be found. This value should be looked up through the data-page's block-indirection table. <blocks ahead...> -- the number of 0x100 byte blocks to move ahead to find the next metadata record (including this metadata). <bytes remaining...>-- pretty straight-forward. So, 0x8# metadata are file-end records; 0xa# are short-range references; and 0xb# are longer range references. This metadata chain provides unamiguous file numbers within the blocks, and since they're put at the block start of any block that contains a file-start, it's easy to pick up from an allocation table reference. Note: 0x8# records don't point to the next metadata block, so we may have to consume file-end padding (0xff until the next multiple of 0x10) if we get an intermediate 0x8# metadata while searching for our target file. """ STATE_NEED_META = 0 STATE_NEED_SKIP_DATA = 1 STATE_NEED_FILE_DATA = 2 STATE_COMPLETE = 3 def __init__(self, file_no, file_len): self.file_no = file_no self.state = MEFileSystemFileMetadataStateMachine.STATE_NEED_META self.bytes_needed = 1 self.byte_offset = 0 self.cur_meta = bytearray(5) self.file_data = bytearray(file_len) self.file_filled = 0 self.found_fileno = False self.work_buf = self.cur_meta def is_complete(self): return self.state == self.STATE_COMPLETE def get_file_data(self): return self.file_data def get_bytes_needed(self): return self.bytes_needed #returns the number of bytes consumed def add_bytes(self, bytes, start_index, data_len=None, log_file = None): """ supplies data to satisfy the state-machine's need for data as reported via get_bytes_needed(). bytes -- the buffer containing the bytes to be fed in start_index -- the start location of the bytes within the buffer data_len -- number of bytes in the array, starting at start_index. if None, then len(bytes) - start_index is assumed """ #shuffling data from potentially multiple calls to fill the data request from the #state machine (get_bytes_needed) data_len = len(bytes) - start_index if data_len is None else data_len if data_len == 0: return 0 # nothing to do #take the min of what's available and what we need to_copy = data_len if data_len < self.bytes_needed else self.bytes_needed if self.work_buf: self.work_buf[self.byte_offset:(self.byte_offset+to_copy)] = bytes[start_index:(start_index+to_copy)] self.byte_offset = self.byte_offset + to_copy self.bytes_needed = self.bytes_needed - to_copy #if we don't have enough to process, return so they can feed more if self.bytes_needed > 0: return to_copy #we only make it this far once we've got the full bytes_needed data meta_type = self.cur_meta[0] & 0xf0 if self.state == self.STATE_NEED_META: if self.byte_offset == 1: if meta_type in [0xa0, 0xb0]: self.bytes_needed = 4 else: self.bytes_needed = 1 else: #Have we found the file number we seek yet? if self.found_fileno or (self.file_no == self.cur_meta[0] & 0x0f): self.found_fileno = True self.state = self.STATE_NEED_FILE_DATA self.work_buf = self.file_data self.byte_offset = self.file_filled else: self.state = self.STATE_NEED_SKIP_DATA self.work_buf = None self.byte_offset = None #determine the data required based on metadata type, and whether we're #skipping (so need to eat EOF padding on type 0x8# entries) or whether #we're copying out file data. if meta_type == 0x80: if self.state == self.STATE_NEED_SKIP_DATA: #if we're skipping a 0x8# entry, we need to eat EOF padding too padding = (0x10 - (self.cur_meta[1] & 0xf)) & 0xf self.bytes_needed = padding + self.cur_meta[1] - 2 #remove header len, too else: self.bytes_needed = self.cur_meta[1] - 2 #remove header len elif meta_type == 0xa0: self.bytes_needed = self.cur_meta[1] - 5 #remove header len elif meta_type == 0xb0: self.bytes_needed = self.cur_meta[1] * 0x100 - 5 #remove header len else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_NEED_SKIP_DATA: #recall: this is the state just *completed* self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 self.bytes_needed = 1 elif self.state == self.STATE_NEED_FILE_DATA: #recall: this is the state just *completed* self.file_filled = self.byte_offset self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 if meta_type == 0x80: #just completed a file-end record...we're done. self.bytes_needed = 0 self.state = self.STATE_COMPLETE elif meta_type in [0xa0, 0xb0]: self.bytes_needed = 1 else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_COMPLETE: #can't leave this state pass else: if log_file: log_file.write("Bad state-machine state: %d\n" % self.state) #recurse to consume as much as we can, for easier calling convention if to_copy < data_len: return to_copy + add_bytes(bytes, start_index+to_copy, data_len-to_copy) #else, return what we consumed return to_copy def read_me_fs_file(file_no, file_len, me_file, log_file = sys.stdout): sm = MEFileSystemFileMetadataStateMachine(file_no, file_len) while not sm.is_complete(): res = sm.add_bytes( bytes=me_file.read(sm.get_bytes_needed()), start_index=0, data_len=None, #shorthand for len(bytes)-start_index log_file=log_file) if not res: log_file.write("Aborting file read.\n") break return sm.get_file_data() class MEFileSystemDataHeader: """ Data Page Header: (Top of a 0x4000-byte data page in the MFS) 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| pgflags | 0x00| 0x00| 0x00| 0x00| mystery bits | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | freed_flags... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ 80 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 90 | block indirection table... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ c0 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ d0+| (file data follows) | +-----------------------------------------------------------------------------------------------+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. pgflags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. myst_bits -- TBD freed_flags-- each bit marks whether a written 0x10-byte chunk is no longer required. Bits exist for the header chunks, as well as the chunks for file data. lsb of the first byte corresponds to the first chunk of the header. 0=freed. blk_itab -- translates ATFileEntry's okey value into a block offset within the data page. offset = blk_itab[okey] * 0x100. This is the location from which to begin the search. Note that an offset of 0 must still respect that the page header consumes bytes [0x00, 0xd0) for the search. (filedata) -- File are padded with 0xff at the end. Note files have internal metadata. See routines above. """ def __init__(self, page_no, flags, bytes_00000000, myst_bits, freed_flags, blk_itab): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.myst_bits = myst_bits self.freed_flags = freed_flags self.blk_itab = blk_itab def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemDataHeader %x:\n" % id(self)) if self.page_no != 0xff: log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" mystery bits: %s\n" % " ".join("%02x"%x for x in self.myst_bits)) log.write(" freed_flags: %s\n" % "".join("%02x"%x for x in self.freed_flags)) log.write(" block ind. tab.: [%s]\n" % " ".join("%02x"%x for x in self.blk_itab)) else: log.write(" (empty)\n") class MEFileSystemATEntry: """ Allocation Table Entry: 0 1 2 3 4 5 6 7 8 9 a +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 |state| flgs| identifier | type| filelen | pgid| okey| fno | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ state -- status of the entry: 0xdc=present; 0xc8=overwritten(i.e., there will be another entry below) flags -- has value 0xf0 in every example...so hard to tell. identifier -- 3-byte identifier...seem to be a preference for ASCII, but there are counterexamples type -- no idea...maybe permissions? Observed values: 0x00, 0x0a, 0x0c, 0x0d, 0x0e, 0x18, 0x1a filelen -- 16-bit, little endian. actual content seems to be a few bytes more...might just be pollution of the structures used to write the file. pgid -- the pgno for the MEFileSystemDataHeader that holds the data. The ATHeader is numbered 0, each data page seems to get sequentially numbered in my examples, though that could change with more use. 0xff indicates "not yet used / nonumber assigned" --- a fact that hints page numbers aren't guaranteed to be sequentially okey -- key for indexing the block_indirection table of the MEFileSystemDataHeader holding this file's content, to find the right 0x100 byte block from which to start the file search according to the 'fno' field. fno -- file number within the block-offset determined from okey. This is to be looked up using the metadata *within* the file data of the data pages. See MEFileSystemFileMetadataStateMachine above """ def __init__(self, state, flags, identifier, type, filelen, pgid, okey, fno): self.state = state self.flags = flags self.identifier = identifier self.type = type self.filelen = filelen self.pgid = pgid self.okey = okey self.fno = fno def debug_print(self, log = sys.stdout): log.write("%15s len=0x%04x [pg=0x%02x k=0x%02x f=0x%02x] ty=0x%02x st=0x%02x, fg=0x%02x" % (str(self.identifier), self.filelen, self.pgid, self.okey, self.fno, self.type, self.state, self.flags)) class MEFileSystemATHeader: """ Allocation Table Header: 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| flags | 0x00| 0x00| 0x00| 0x00|"MFS\0" |bitfields? | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | bitfields? | +-----+-----+-----+-----+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. flags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. "MFS\0" -- ASCIIZ signature for the MFS AT Header bitfields -- 64 bits of apparent bitfields """ max_files = int((0x4000 - 0x14) / 11) #page size, less the header, divided by file entry size def __init__(self, page_no, flags, bytes_00000000, sig, bitfields): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.sig_good = sig == b'MFS\x00' self.bitfields = bitfields def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemATHeader %x:\n" % id(self)) log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" sig good: %s\n" % str(self.sig_good)) log.write(" bitfields: %s\n" % " ".join("%02x"%x for x in self.bitfields)) class MEFileSystemAT: """ Allocation Table (a 0x4000-byte page in the MFS): 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | Allocation Table Header... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 |... | File Entry |FE...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ ...|... | +-----------------------------------------------------------------------------------+-----+-----+ 3fe0|... |File Ent...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 3ff0|... |Padding | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ """ def __init__(self, header, entry_list): self.header = header self.entry_list = entry_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemAT %x:\n" % id(self)) self.header.debug_print(log) log.write("File entries:\n") for ent in self.entry_list: if(ent): log.write(" ") ent.debug_print(log) log.write("\n") class MEFileSystem: """ MFS Partition: (Making the assumption the AT is the first page) +------------------------------------+ 0000 | Allocation Table | +------------------------------------+ 4000 | Data Page | +------------------------------------+ 8000 | Data Page | +------------------------------------+ c000 | Data Page | +------------------------------------+ ... |... | +------------------------------------+ """ def __init__(self, allocation_table, data_page_list): self.allocation_table = allocation_table self.data_page_list = data_page_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystem %x:\n" % id(self)) self.allocation_table.debug_print(log) for page in self.data_page_list: if(page): page.debug_print(log) else: log.write("(Not parsed)\n") def parse_me_fs_at_entry(me_file, log_file = sys.stdout): return MEFileSystemATEntry( state = read_leuint8(me_file), flags = read_leuint8(me_file), identifier = me_file.read(3), type = read_leuint8(me_file), filelen = read_leuint16(me_file), pgid = read_leuint8(me_file), okey = read_leuint8(me_file), fno = read_leuint8(me_file)) def parse_me_fs_at_header(me_file, log_file = sys.stdout): return MEFileSystemATHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), sig = me_file.read(4), bitfields = me_file.read(8)) def parse_me_fs_at(me_file, log_file = sys.stdout): hdr = parse_me_fs_at_header(me_file, log_file) entry_list = [None] * MEFileSystemATHeader.max_files for i in range(MEFileSystemATHeader.max_files): ent = parse_me_fs_at_entry(me_file, log_file) if ent.state == 0xff: break entry_list[i] = ent return MEFileSystemAT(hdr, entry_list) def parse_me_fs_data_header(me_file, log_file = sys.stdout): return MEFileSystemDataHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), myst_bits = me_file.read(0x8), freed_flags = me_file.read(0x80), blk_itab = me_file.read(0x40)) def parse_me_fs(length, me_file, log_file = sys.stdout): """ ARGS: length -- length in bytes of the ME partition holding the MFS/MFSB data me_file -- a file handle whose present position is the start of the MFS(B) partition log_file -- if there is diagnostic output, put it here RETURNS: an MEFileSystem instance populated with the allocation table and data-page headers """ total_pages = int(length/0x4000) start_offset = me_file.tell() #NOTE: I'm presuming the allocation table is the first page... #that might not be reliable... at = parse_me_fs_at(me_file, log_file) data_pages = [None] * (total_pages-1) for page in range(0, total_pages-1): me_file.seek(start_offset + 0x4000 * (page+1)) data_pages[page] = parse_me_fs_data_header(me_file, log_file) return MEFileSystem( allocation_table=at, data_page_list = data_pages) def get_mfs_file(mefs, me_file, mfs_file_offset, id, log_file = sys.stdout): """ Example of how to use the MEFileSystem structures to retrieve MFS and MFSB files. ARGS: mefs -- a MEFileSystem instance parsed from mfs_data me_file -- a filehandle containing the ME image mfs_file_offset -- the file offset within me_file where the MFS partition begins id -- a 3-byte byte array with the file identifier log -- if there is diagnostic output, put it here RETURNS: an array containing [state, The data from the corresponding file]. else None if the file identifier does not exist within the data. Example driver, given the known offset and size for a MFS partition: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) result_tuple = get_mfs_file(mefs, spi_image_file, MFS_OFFSET, b'UKS') if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple)) """ #Find the file identifer in the Allocation Table best_ent = None for ent in mefs.allocation_table.entry_list: if ent and ent.identifier == id: best_ent = ent if ent.state == 0xdc: break; # got a current entry log.write("Error: found an item w/ state %02x...continuing\n" % ent.state) #if found, lookup which data page matches the entry's pgid value if best_ent: page_found = False for list_idx in range(len(mefs.data_page_list)): page = mefs.data_page_list[list_idx] if page.page_no == best_ent.pgid: page_found = True #we found the right data page, so start the file search search_start = page.blk_itab[best_ent.okey] * 0x100 #In the following lines: # The value d0 is to skip over the datapage header if we're in the first block # # The multiple of 0x4000 selects the data offset that goes with list_idx # since the parsed data-page list is in the same order as found in the file. # # Because mefs.data_page_list doesn't include the allocation table page, we +1 # to the index before multiplying. The result is a set of offsets into the MFS data # bounding the file search ## search_off = 0x4000 * (list_idx+1) + (0xd0 if search_start == 0 else search_start) me_file.seek(mfs_file_offset + search_off) data = read_me_fs_file(best_ent.fno, best_ent.filelen, me_file, log_file) if data: return [best_ent.state, data] return None if __name__ == "__main__": with open("image.bin", "rb") as spi_image_file: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) #Dump the allocation table mefs.allocation_table.debug_print() print("") first_file = mefs.allocation_table.entry_list[0].identifier print("looking up the first file (%s):" % first_file) result_tuple = get_mfs_file(mefs, spi_image_file, MFS_OFFSET, first_file) if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple))

learned a little more about metadata in the files, added to the code.

Source Link

edited Apr 6, 2020 at 9:33

lockcmpxchg8b

import sys #litte-endian integer readers def read_leuint8(file): data = file.read(1) return data[0] def read_leuint16(file): data = file.read(2) return data[0] | (data[1] << 8) def read_leuint24(file): data = file.read(3) return data[0] | (data[1] << 8) | (data[2] << 16) def read_leuint32(file): data = file.read(4) return data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24) class MEFileSystemFileMetadataStateMachine: """ MEFileSystemFileMetadata: Files in MFS have internal metadata entries. Each file begins with a metadata record, observed values are: 0xa# <bytes ahead until next metadata> <0x01> <next meta block num> <0x00> 0xb# <blocks ahead until next metadata> <0x01> <next meta block num> <0x00> 0x8# <bytes remaining in file (including this metadata)> where # -- is the file number within the block. This is the target for the fno field of the allocation table entry. <bytes ahead...> -- the number of bytes to move ahead to find the next metadata record (including this metadata). <next meta block...>-- the 0x100-byte block number where the next metadata record is to be found. This value should be looked up through the data-page's block-indirection table. <blocks ahead...> -- the number of 0x100 byte blocks to move ahead to find the next metadata record (including this metadata). <bytes remaining...>-- pretty straight-forward. So, 0x8# metadata are file-end records; 0xa# are short-range references; and 0xb# are longer range references. This metadata chain provides unamiguous file numbers within the blocks, and since they're put at the block start of any block that contains a file-start, it's easy to pick up from an allocation table reference. Note: 0x8# records don't point to the next metadata block, so we may have to consume file-end padding (0xff until the next multiple of 0x10) if we get an intermediate 0x8# metadata while searching for our target file. """ STATE_NEED_META = 0 STATE_NEED_SKIP_DATA = 1 STATE_NEED_FILE_DATA = 2 STATE_COMPLETE = 3 def __init__(self, file_no, file_len): self.file_no = file_no self.state = MEFileSystemFileMetadataStateMachine.STATE_NEED_META self.bytes_needed = 1 self.byte_offset = 0 self.cur_meta = bytearray(5) self.file_data = bytearray(file_len) self.file_filled = 0 self.found_fileno = False self.work_buf = self.cur_meta def is_complete(self): return self.state == self.STATE_COMPLETE def get_file_data(self): return self.file_data def get_bytes_needed(self): return self.bytes_needed #returns the number of bytes consumed def add_bytes(self, bytes, start_index, data_len=None, log_file = None): """ supplies data to satisfy the state-machine's need for data as reported via get_bytes_needed(). bytes -- the buffer containing the bytes to be fed in start_index -- the start location of the bytes within the buffer data_len -- number of bytes in the array, starting at start_index. if None, then len(bytes) - start_index is assumed """ #shuffling data from potentially multiple calls to fill the data request from the #state machine (get_bytes_needed) data_len = len(bytes) - start_index if data_len is None else data_len if data_len == 0: return 0 # nothing to do #take the min of what's available and what we need to_copy = data_len if data_len < self.bytes_needed else self.bytes_needed if self.work_buf: self.work_buf[self.byte_offset:(self.byte_offset+to_copy)] = bytes[start_index:(start_index+to_copy)] self.byte_offset = self.byte_offset + to_copy self.bytes_needed = self.bytes_needed - to_copy #if we don't have enough to process, return so they can feed more if self.bytes_needed > 0: return to_copy #we only make it this far once we've got the full bytes_needed data meta_type = self.cur_meta[0] & 0xf0 if self.state == self.STATE_NEED_META: if self.byte_offset == 1: if meta_type in [0xa0, 0xb0]: self.bytes_needed = 4 else: self.bytes_needed = 1 else: #Have we found the file number we seek yet? if self.found_fileno or (self.file_no == self.cur_meta[0] & 0x0f): self.found_fileno = True self.state = self.STATE_NEED_FILE_DATA self.work_buf = self.file_data self.byte_offset = self.file_filled else: self.state = self.STATE_NEED_SKIP_DATA self.work_buf = None self.byte_offset = None #determine the data required based on metadata type, and whether we're #skipping (so need to eat EOF padding on type 0x8# entries) or whether #we're copying out file data. if meta_type == 0x80: if self.state == self.STATE_NEED_SKIP_DATA: #if we're skipping a 0x8# entry, we need to eat EOF padding too padding = (0x10 - (self.cur_meta[1] & 0xf)) & 0xf self.bytes_needed = padding + self.cur_meta[1] - 2 #remove header len, too else: self.bytes_needed = self.cur_meta[1] - 2 #remove header len elif meta_type == 0xa0: self.bytes_needed = self.cur_meta[1] - 5 #remove header len elif meta_type == 0xb0: self.bytes_needed = self.cur_meta[1] * 0x100 - 5 #remove header len else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_NEED_SKIP_DATA: #recall: this is the state just *completed* self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 self.bytes_needed = 1 elif self.state == self.STATE_NEED_FILE_DATA: #recall: this is the state just *completed* self.file_filled = self.byte_offset self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 if meta_type == 0x80: #just completed a file-end record...we're done. self.bytes_needed = 0 self.state = self.STATE_COMPLETE elif meta_type in [0xa0, 0xb0]: self.bytes_needed = 1 else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_COMPLETE: #can't leave this state pass else: if log_file: log_file.write("Bad state-machine state: %d\n" % self.state) #recurse to consume as much as we can, for easier calling convention if to_copy < data_len: return to_copy + add_bytes(bytes, start_index+to_copy, data_len-to_copy) #else, return what we consumed return to_copy def read_me_fs_file(file_no, file_len, me_file, log_file = sys.stdout): sm = MEFileSystemFileMetadataStateMachine(file_no, file_len) while not sm.is_complete(): res = sm.add_bytes( bytes=me_file.read(sm.get_bytes_needed()), start_index=0, data_len=None, #shorthand for len(bytes)-start_index log_file=log_file) if not res: log_file.write("Aborting file read.\n") break return sm.get_file_data() class MEFileSystemDataHeader: """ Data Page Header: (Top of a 0x4000-byte data page in the MFS) 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| pgflags | 0x00| 0x00| 0x00| 0x00| mystery bits | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | freed_flags... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ 80 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 90 | block indirection table... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ c0 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ d0+| (file data follows) | +-----------------------------------------------------------------------------------------------+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. pgflags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. myst_bits -- TBD freed_flags-- each bit marks whether a written 0x10-byte chunk is no longer required. Bits exist for the header chunks, as well as the chunks for file data. lsb of the first byte corresponds to the first chunk of the header. 0=freed. blk_itab -- translates ATFileEntry's okey value into a block offset within the data page. offset = blk_itab[okey] * 0x100. This is the location from which to begin the search. Note that an offset of 0 must still respect that the page header consumes bytes [0x00, 0xd0) for the search. (filedata) -- File are padded with 0xff at the end. Note files seem to have at least 2 bytes more data than the specified length. Larger files have more. """ def __init__(self, page_no, flags, bytes_00000000, myst_bits, freed_flags, blk_itab): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.myst_bits = myst_bits self.freed_flags = freed_flags self.blk_itab = blk_itab def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemDataHeader %x:\n" % id(self)) if self.page_no != 0xff: log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" mystery bits: %s\n" % " ".join("%02x"%x for x in self.myst_bits)) log.write(" freed_flags: %s\n" % "".join("%02x"%x for x in self.freed_flags)) log.write(" block ind. tab.: [%s]\n" % " ".join("%02x"%x for x in self.blk_itab)) else: log.write(" (empty)\n") class MEFileSystemATEntry: """ Allocation Table Entry: 0 1 2 3 4 5 6 7 8 9 a +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 |state| flgs| identifier | type| filelen | pgid| okey| fno | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ state -- status of the entry: 0xdc=present; 0xc8=overwritten(i.e., there will be another entry below) flags -- has value 0xf0 in every example...so hard to tell. identifier -- 3-byte identifier...seem to be a preference for ASCII, but there are counterexamples type -- no idea...maybe permissions? Observed values: 0x00, 0x0a, 0x0c, 0x0d, 0x0e, 0x18, 0x1a filelen -- 16-bit, little endian. actual content seems to be a few bytes more...might just be pollution of the structures used to write the file. pgid -- the pgno for the MEFileSystemDataHeader that holds the data. The ATHeader is numbered 0, each data page seems to get sequentially numbered in my examples, though that could change with more use. 0xff indicates "not yet used / nonumber assigned" --- a fact that hints page numbers aren't guaranteed to be sequentially okey -- key for indexing the block_indirection table of the MEFileSystemDataHeader holding this file's content, to find the right 0x100 byte block from which to start the file search according to the 'fno' field. fno -- file number within the block-offset determined from okey. This is to be looked up using the metadata *within* the file data of the data pages. See MEFileSystemFileMetadataStateMachine above """ def __init__(self, state, flags, identifier, type, filelen, pgid, okey, fno): self.state = state self.flags = flags self.identifier = identifier self.type = type self.filelen = filelen self.pgid = pgid self.okey = okey self.fno = fno def debug_print(self, log = sys.stdout): log.write("%15s len=0x%04x [pg=0x%02x k=0x%02x f=0x%02x] ty=0x%02x st=0x%02x, fg=0x%02x" % (str(self.identifier), self.filelen, self.pgid, self.okey, self.fno, self.type, self.state, self.flags)) class MEFileSystemATHeader: """ Allocation Table Header: 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| flags | 0x00| 0x00| 0x00| 0x00|"MFS\0" |bitfields? | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | bitfields? | +-----+-----+-----+-----+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. flags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. "MFS\0" -- ASCIIZ signature for the MFS AT Header bitfields -- 64 bits of apparent bitfields """ max_files = int((0x4000 - 0x14) / 11) #page size, less the header, divided by file entry size def __init__(self, page_no, flags, bytes_00000000, sig, bitfields): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.sig_good = sig == b'MFS\x00' self.bitfields = bitfields def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemATHeader %x:\n" % id(self)) log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" sig good: %s\n" % str(self.sig_good)) log.write(" bitfields: %s\n" % " ".join("%02x"%x for x in self.bitfields)) class MEFileSystemAT: """ Allocation Table (a 0x4000-byte page in the MFS): 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | Allocation Table Header... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 |... | File Entry |FE...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ ...|... | +-----------------------------------------------------------------------------------+-----+-----+ 3fe0|... |File Ent...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 3ff0|... |Padding | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ """ def __init__(self, header, entry_list): self.header = header self.entry_list = entry_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemAT %x:\n" % id(self)) self.header.debug_print(log) log.write("File entries:\n") for ent in self.entry_list: if(ent): log.write(" ") ent.debug_print(log) log.write("\n") class MEFileSystem: """ MFS Partition: (Making the assumption the AT is the first page) +------------------------------------+ 0000 | Allocation Table | +------------------------------------+ 4000 | Data Page | +------------------------------------+ 8000 | Data Page | +------------------------------------+ c000 | Data Page | +------------------------------------+ ... |... | +------------------------------------+ """ def __init__(self, allocation_table, data_page_list): self.allocation_table = allocation_table self.data_page_list = data_page_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystem %x:\n" % id(self)) self.allocation_table.debug_print(log) for page in self.data_page_list: if(page): page.debug_print(log) else: log.write("(Not parsed)\n") def parse_me_fs_at_entry(me_file, log_file = sys.stdout): return MEFileSystemATEntry( state = read_leuint8(me_file), flags = read_leuint8(me_file), identifier = me_file.read(3), type = read_leuint8(me_file), filelen = read_leuint16(me_file), pgid = read_leuint8(me_file), okey = read_leuint8(me_file), fno = read_leuint8(me_file)) def parse_me_fs_at_header(me_file, log_file = sys.stdout): return MEFileSystemATHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), sig = me_file.read(4), bitfields = me_file.read(8)) def parse_me_fs_at(me_file, log_file = sys.stdout): hdr = parse_me_fs_at_header(me_file, log_file) entry_list = [None] * MEFileSystemATHeader.max_files for i in range(MEFileSystemATHeader.max_files): ent = parse_me_fs_at_entry(me_file, log_file) if ent.state == 0xff: break entry_list[i] = ent return MEFileSystemAT(hdr, entry_list) def parse_me_fs_data_header(me_file, log_file = sys.stdout): return MEFileSystemDataHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), myst_bits = me_file.read(0x8), freed_flags = me_file.read(0x80), blk_itab = me_file.read(0x40)) def parse_me_fs(length, me_file, log_file = sys.stdout): """ ARGS: length -- length in bytes of the ME partition holding the MFS/MFSB data me_file -- a file handle whose present position is the start of the MFS(B) partition log_file -- if there is diagnostic output, put it here RETURNS: an MEFileSystem instance populated with the allocation table and data-page headers """ total_pages = int(length/0x4000) start_offset = me_file.tell() #NOTE: I'm presuming the allocation table is the first page... #that might not be reliable... at = parse_me_fs_at(me_file, log_file) data_pages = [None] * (total_pages-1) for page in range(0, total_pages-1): me_file.seek(start_offset + 0x4000 * (page+1)) data_pages[page] = parse_me_fs_data_header(me_file, log_file) return MEFileSystem( allocation_table=at, data_page_list = data_pages) def get_mfs_file(mefs, me_file, mfs_file_offset, id, log_file = sys.stdout): """ Example of how to use the MEFileSystem structures to retrieve MFS and MFSB files. ARGS: mefs -- a MEFileSystem instance parsed from mfs_data me_file -- a filehandle containing the ME image mfs_file_offset -- the file offset within me_file where the MFS partition begins id -- a 3-byte byte array with the file identifier log -- if there is diagnostic output, put it here RETURNS: an array containing [state, The data from the corresponding file]. else None if the file identifier does not exist within the data. Example driver, given the known offset and size for a MFS partition: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) result_tuple = get_mfs_file(mefs, spi_image_file, MFS_OFFSET, b'UKS') if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple)) """ #Find the file identifer in the Allocation Table best_ent = None for ent in mefs.allocation_table.entry_list: if ent and ent.identifier == id: best_ent = ent if ent.state == 0xdc: break; # got a current entry log.write("Error: found an item w/ state %02x...continuing\n" % ent.state) #if found, lookup which data page matches the entry's pgid value if best_ent: page_found = False for list_idx in range(len(mefs.data_page_list)): page = mefs.data_page_list[list_idx] if page.page_no == best_ent.pgid: page_found = True #we found the right data page, so start the file search search_start = page.blk_itab[best_ent.okey] * 0x100 #In the following lines: # The value d0 is to skip over the datapage header if we're in the first block # # The multiple of 0x4000 selects the data offset that goes with list_idx # since the parsed data-page list is in the same order as found in the file. # # Because mefs.data_page_list doesn't include the allocation table page, we +1 # to the index before multiplying. The result is a set of offsets into the MFS data # bounding the file search ## search_off = 0x4000 * (list_idx+1) + (0xd0 if search_start == 0 else search_start) me_file.seek(mfs_file_offset + search_off) data = read_me_fs_file(best_ent.fno, best_ent.filelen, me_file, log_file) if data: return [best_ent.state, data] return None if __name__ == "__main__": with open("image.bin", "rb") as spi_image_file: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) #Dump the allocation table mefs.allocation_table.debug_print() print("")   first_file = mefs.allocation_table.entry_list[0].identifier print("looking up the first file (%s):" % mefs.allocation_table.entry_list[0].identifierfirst_file) result_tuple = get_mfs_file(mefs, spi_image_file, MFS_OFFSET, b'UKS'first_file) if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple))

 b'UKS' len=0x0004 [pg=0x01 k=0x00 s=0x00]f=0x00] ty=0x0a st=0xdc, fg=0xf0 b'LRT' len=0x01c0 [pg=0x01 k=0x00 s=0x01]f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'MIA' len=0x0003 [pg=0x01 k=0x02 s=0x01]f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'BLV' len=0x0004 [pg=0x01 k=0x02 s=0x02]f=0x02] ty=0x0a st=0xdc, fg=0xf0 b'SDV' len=0x00ff [pg=0x01 k=0x02 s=0x03]f=0x03] ty=0x0a st=0xdc, fg=0xf0 b'ICP' len=0x0042 [pg=0x01 k=0x03 s=0x01]f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'\x00NP' len=0x0001 [pg=0x01 k=0x04 s=0x01]f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'PPN' len=0x0042 [pg=0x01 k=0x04 s=0x02]f=0x02] ty=0x0e st=0xdc, fg=0xf0 b'SCO' len=0x00af [pg=0x01 k=0x04 s=0x03]f=0x03] ty=0x0a st=0xdc, fg=0xf0 b'PCO' len=0x0a24 [pg=0x01 k=0x05 s=0x01]f=0x01] ty=0x0a st=0xdc, fg=0xf0 b'GFC' len=0x0004 [pg=0x01 k=0x07 s=0x01]f=0x01] ty=0x18 st=0xdc, fg=0xf0 b'YHP' len=0x06fe [pg=0x01 k=0x07 s=0x02]f=0x02] ty=0x0a st=0xdc, fg=0xf0 b'FCP' len=0x0002 [pg=0x01 k=0x09 s=0x01]f=0x01] ty=0x0b st=0xdc, fg=0xf0 b'PPR' len=0x0001 [pg=0x01 k=0x09 s=0x02]f=0x02] ty=0x0b st=0xdc, fg=0xf0 b'TCM' len=0x0005 [pg=0x01 k=0x09 s=0x03]f=0x03] ty=0x10 st=0xdc, fg=0xf0 b'BHM' len=0x0004 [pg=0x01 k=0x09 s=0x04]f=0x04] ty=0x10 st=0xdc, fg=0xf0 b'GCN' len=0x0018 [pg=0x01 k=0x09 s=0x05]f=0x05] ty=0x0e st=0xdc, fg=0xf0 b'CSM' len=0x000f [pg=0x01 k=0x0a s=0x00]f=0x00] ty=0x0e st=0xdc, fg=0xf0 b'\x00HS' len=0x0127 [pg=0x01 k=0x0a s=0x01]f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'\x01HS' len=0x0127 [pg=0x01 k=0x0b s=0x01]f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'\x02HS' len=0x0127 [pg=0x01 k=0x0c s=0x01]f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'\x03HS' len=0x0127 [pg=0x01 k=0x0d s=0x01]f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'PCF' len=0x0010 [pg=0x01 k=0x0e s=0x01]f=0x01] ty=0x0e st=0xdc, fg=0xf0 ...

import sys #litte-endian integer readers def read_leuint8(file): data = file.read(1) return data[0] def read_leuint16(file): data = file.read(2) return data[0] | (data[1] << 8) def read_leuint24(file): data = file.read(3) return data[0] | (data[1] << 8) | (data[2] << 16) def read_leuint32(file): data = file.read(4) return data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24) class MEFileSystemFileMetadataStateMachine: """ MEFileSystemFileMetadata: Files in MFS have internal metadata entries. Each file begins with a metadata record, observed values are: 0xa# <bytes ahead until next metadata> <0x01> <next meta block num> <0x00> 0xb# <blocks ahead until next metadata> <0x01> <next meta block num> <0x00> 0x8# <bytes remaining in file (including this metadata)> where # -- is the file number within the block. This is the target for the fno field of the allocation table entry. <bytes ahead...> -- the number of bytes to move ahead to find the next metadata record (including this metadata). <next meta block...>-- the 0x100-byte block number where the next metadata record is to be found. This value should be looked up through the data-page's block-indirection table. <blocks ahead...> -- the number of 0x100 byte blocks to move ahead to find the next metadata record (including this metadata). <bytes remaining...>-- pretty straight-forward. So, 0x8# metadata are file-end records; 0xa# are short-range references; and 0xb# are longer range references. This metadata chain provides unamiguous file numbers within the blocks, and since they're put at the block start of any block that contains a file-start, it's easy to pick up from an allocation table reference. Note: 0x8# records don't point to the next metadata block, so we may have to consume file-end padding (0xff until the next multiple of 0x10) if we get an intermediate 0x8# metadata while searching for our target file. """ STATE_NEED_META = 0 STATE_NEED_SKIP_DATA = 1 STATE_NEED_FILE_DATA = 2 STATE_COMPLETE = 3 def __init__(self, file_no, file_len): self.file_no = file_no self.state = MEFileSystemFileMetadataStateMachine.STATE_NEED_META self.bytes_needed = 1 self.byte_offset = 0 self.cur_meta = bytearray(5) self.file_data = bytearray(file_len) self.file_filled = 0 self.found_fileno = False self.work_buf = self.cur_meta def is_complete(self): return self.state == self.STATE_COMPLETE def get_file_data(self): return self.file_data def get_bytes_needed(self): return self.bytes_needed #returns the number of bytes consumed def add_bytes(self, bytes, start_index, data_len=None, log_file = None): """ supplies data to satisfy the state-machine's need for data as reported via get_bytes_needed(). bytes -- the buffer containing the bytes to be fed in start_index -- the start location of the bytes within the buffer data_len -- number of bytes in the array, starting at start_index. if None, then len(bytes) - start_index is assumed """ #shuffling data from potentially multiple calls to fill the data request from the #state machine (get_bytes_needed) data_len = len(bytes) - start_index if data_len is None else data_len if data_len == 0: return 0 # nothing to do #take the min of what's available and what we need to_copy = data_len if data_len < self.bytes_needed else self.bytes_needed if self.work_buf: self.work_buf[self.byte_offset:(self.byte_offset+to_copy)] = bytes[start_index:(start_index+to_copy)] self.byte_offset = self.byte_offset + to_copy self.bytes_needed = self.bytes_needed - to_copy #if we don't have enough to process, return so they can feed more if self.bytes_needed > 0: return to_copy #we only make it this far once we've got the full bytes_needed data meta_type = self.cur_meta[0] & 0xf0 if self.state == self.STATE_NEED_META: if self.byte_offset == 1: if meta_type in [0xa0, 0xb0]: self.bytes_needed = 4 else: self.bytes_needed = 1 else: #Have we found the file number we seek yet? if self.found_fileno or (self.file_no == self.cur_meta[0] & 0x0f): self.found_fileno = True self.state = self.STATE_NEED_FILE_DATA self.work_buf = self.file_data self.byte_offset = self.file_filled else: self.state = self.STATE_NEED_SKIP_DATA self.work_buf = None self.byte_offset = None #determine the data required based on metadata type, and whether we're #skipping (so need to eat EOF padding on type 0x8# entries) or whether #we're copying out file data. if meta_type == 0x80: if self.state == self.STATE_NEED_SKIP_DATA: #if we're skipping a 0x8# entry, we need to eat EOF padding too padding = (0x10 - (self.cur_meta[1] & 0xf)) & 0xf self.bytes_needed = padding + self.cur_meta[1] - 2 #remove header len, too else: self.bytes_needed = self.cur_meta[1] - 2 #remove header len elif meta_type == 0xa0: self.bytes_needed = self.cur_meta[1] - 5 #remove header len elif meta_type == 0xb0: self.bytes_needed = self.cur_meta[1] * 0x100 - 5 #remove header len else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_NEED_SKIP_DATA: #recall: this is the state just *completed* self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 self.bytes_needed = 1 elif self.state == self.STATE_NEED_FILE_DATA: #recall: this is the state just *completed* self.file_filled = self.byte_offset self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 if meta_type == 0x80: #just completed a file-end record...we're done. self.bytes_needed = 0 self.state = self.STATE_COMPLETE elif meta_type in [0xa0, 0xb0]: self.bytes_needed = 1 else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_COMPLETE: #can't leave this state pass else: if log_file: log_file.write("Bad state-machine state: %d\n" % self.state) #recurse to consume as much as we can, for easier calling convention if to_copy < data_len: return to_copy + add_bytes(bytes, start_index+to_copy, data_len-to_copy) #else, return what we consumed return to_copy def read_me_fs_file(file_no, file_len, me_file, log_file = sys.stdout): sm = MEFileSystemFileMetadataStateMachine(file_no, file_len) while not sm.is_complete(): res = sm.add_bytes( bytes=me_file.read(sm.get_bytes_needed()), start_index=0, data_len=None, #shorthand for len(bytes)-start_index log_file=log_file) if not res: log_file.write("Aborting file read.\n") break return sm.get_file_data() class MEFileSystemDataHeader: """ Data Page Header: (Top of a 0x4000-byte data page in the MFS) 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| pgflags | 0x00| 0x00| 0x00| 0x00| mystery bits | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | freed_flags... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ 80 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 90 | block indirection table... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ c0 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ d0+| (file data follows) | +-----------------------------------------------------------------------------------------------+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. pgflags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. myst_bits -- TBD freed_flags-- each bit marks whether a written 0x10-byte chunk is no longer required. Bits exist for the header chunks, as well as the chunks for file data. lsb of the first byte corresponds to the first chunk of the header. 0=freed. blk_itab -- translates ATFileEntry's okey value into a block offset within the data page. offset = blk_itab[okey] * 0x100. This is the location from which to begin the search. Note that an offset of 0 must still respect that the page header consumes bytes [0x00, 0xd0) for the search. (filedata) -- File are padded with 0xff at the end. Note files seem to have at least 2 bytes more data than the specified length. Larger files have more. """ def __init__(self, page_no, flags, bytes_00000000, myst_bits, freed_flags, blk_itab): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.myst_bits = myst_bits self.freed_flags = freed_flags self.blk_itab = blk_itab def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemDataHeader %x:\n" % id(self)) if self.page_no != 0xff: log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" mystery bits: %s\n" % " ".join("%02x"%x for x in self.myst_bits)) log.write(" freed_flags: %s\n" % "".join("%02x"%x for x in self.freed_flags)) log.write(" block ind. tab.: [%s]\n" % " ".join("%02x"%x for x in self.blk_itab)) else: log.write(" (empty)\n") class MEFileSystemATEntry: """ Allocation Table Entry: 0 1 2 3 4 5 6 7 8 9 a +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 |state| flgs| identifier | type| filelen | pgid| okey| fno | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ state -- status of the entry: 0xdc=present; 0xc8=overwritten(i.e., there will be another entry below) flags -- has value 0xf0 in every example...so hard to tell. identifier -- 3-byte identifier...seem to be a preference for ASCII, but there are counterexamples type -- no idea...maybe permissions? Observed values: 0x00, 0x0a, 0x0c, 0x0d, 0x0e, 0x18, 0x1a filelen -- 16-bit, little endian. actual content seems to be a few bytes more...might just be pollution of the structures used to write the file. pgid -- the pgno for the MEFileSystemDataHeader that holds the data. The ATHeader is numbered 0, each data page seems to get sequentially numbered in my examples, though that could change with more use. 0xff indicates "not yet used / nonumber assigned" --- a fact that hints page numbers aren't guaranteed to be sequentially okey -- key for indexing the block_indirection table of the MEFileSystemDataHeader holding this file's content, to find the right 0x100 byte block from which to start the file search according to the 'fno' field. fno -- file number within the block-offset determined from okey. This is to be looked up using the metadata *within* the file data of the data pages. See MEFileSystemFileMetadataStateMachine above """ def __init__(self, state, flags, identifier, type, filelen, pgid, okey, fno): self.state = state self.flags = flags self.identifier = identifier self.type = type self.filelen = filelen self.pgid = pgid self.okey = okey self.fno = fno def debug_print(self, log = sys.stdout): log.write("%15s len=0x%04x [pg=0x%02x k=0x%02x f=0x%02x] ty=0x%02x st=0x%02x, fg=0x%02x" % (str(self.identifier), self.filelen, self.pgid, self.okey, self.fno, self.type, self.state, self.flags)) class MEFileSystemATHeader: """ Allocation Table Header: 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| flags | 0x00| 0x00| 0x00| 0x00|"MFS\0" |bitfields? | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | bitfields? | +-----+-----+-----+-----+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. flags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. "MFS\0" -- ASCIIZ signature for the MFS AT Header bitfields -- 64 bits of apparent bitfields """ max_files = int((0x4000 - 0x14) / 11) #page size, less the header, divided by file entry size def __init__(self, page_no, flags, bytes_00000000, sig, bitfields): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.sig_good = sig == b'MFS\x00' self.bitfields = bitfields def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemATHeader %x:\n" % id(self)) log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" sig good: %s\n" % str(self.sig_good)) log.write(" bitfields: %s\n" % " ".join("%02x"%x for x in self.bitfields)) class MEFileSystemAT: """ Allocation Table (a 0x4000-byte page in the MFS): 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | Allocation Table Header... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 |... | File Entry |FE...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ ...|... | +-----------------------------------------------------------------------------------+-----+-----+ 3fe0|... |File Ent...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 3ff0|... |Padding | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ """ def __init__(self, header, entry_list): self.header = header self.entry_list = entry_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemAT %x:\n" % id(self)) self.header.debug_print(log) log.write("File entries:\n") for ent in self.entry_list: if(ent): log.write(" ") ent.debug_print(log) log.write("\n") class MEFileSystem: """ MFS Partition: (Making the assumption the AT is the first page) +------------------------------------+ 0000 | Allocation Table | +------------------------------------+ 4000 | Data Page | +------------------------------------+ 8000 | Data Page | +------------------------------------+ c000 | Data Page | +------------------------------------+ ... |... | +------------------------------------+ """ def __init__(self, allocation_table, data_page_list): self.allocation_table = allocation_table self.data_page_list = data_page_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystem %x:\n" % id(self)) self.allocation_table.debug_print(log) for page in self.data_page_list: if(page): page.debug_print(log) else: log.write("(Not parsed)\n") def parse_me_fs_at_entry(me_file, log_file = sys.stdout): return MEFileSystemATEntry( state = read_leuint8(me_file), flags = read_leuint8(me_file), identifier = me_file.read(3), type = read_leuint8(me_file), filelen = read_leuint16(me_file), pgid = read_leuint8(me_file), okey = read_leuint8(me_file), fno = read_leuint8(me_file)) def parse_me_fs_at_header(me_file, log_file = sys.stdout): return MEFileSystemATHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), sig = me_file.read(4), bitfields = me_file.read(8)) def parse_me_fs_at(me_file, log_file = sys.stdout): hdr = parse_me_fs_at_header(me_file, log_file) entry_list = [None] * MEFileSystemATHeader.max_files for i in range(MEFileSystemATHeader.max_files): ent = parse_me_fs_at_entry(me_file, log_file) if ent.state == 0xff: break entry_list[i] = ent return MEFileSystemAT(hdr, entry_list) def parse_me_fs_data_header(me_file, log_file = sys.stdout): return MEFileSystemDataHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), myst_bits = me_file.read(0x8), freed_flags = me_file.read(0x80), blk_itab = me_file.read(0x40)) def parse_me_fs(length, me_file, log_file = sys.stdout): """ ARGS: length -- length in bytes of the ME partition holding the MFS/MFSB data me_file -- a file handle whose present position is the start of the MFS(B) partition log_file -- if there is diagnostic output, put it here RETURNS: an MEFileSystem instance populated with the allocation table and data-page headers """ total_pages = int(length/0x4000) start_offset = me_file.tell() #NOTE: I'm presuming the allocation table is the first page... #that might not be reliable... at = parse_me_fs_at(me_file, log_file) data_pages = [None] * (total_pages-1) for page in range(0, total_pages-1): me_file.seek(start_offset + 0x4000 * (page+1)) data_pages[page] = parse_me_fs_data_header(me_file, log_file) return MEFileSystem( allocation_table=at, data_page_list = data_pages) def get_mfs_file(mefs, me_file, mfs_file_offset, id, log_file = sys.stdout): """ Example of how to use the MEFileSystem structures to retrieve MFS and MFSB files. ARGS: mefs -- a MEFileSystem instance parsed from mfs_data me_file -- a filehandle containing the ME image mfs_file_offset -- the file offset within me_file where the MFS partition begins id -- a 3-byte byte array with the file identifier log -- if there is diagnostic output, put it here RETURNS: an array containing [state, The data from the corresponding file]. else None if the file identifier does not exist within the data. Example driver, given the known offset and size for a MFS partition: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) result_tuple = get_mfs_file(mefs, spi_image_file, MFS_OFFSET, b'UKS') if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple)) """ #Find the file identifer in the Allocation Table best_ent = None for ent in mefs.allocation_table.entry_list: if ent and ent.identifier == id: best_ent = ent if ent.state == 0xdc: break; # got a current entry log.write("Error: found an item w/ state %02x...continuing\n" % ent.state) #if found, lookup which data page matches the entry's pgid value if best_ent: page_found = False for list_idx in range(len(mefs.data_page_list)): page = mefs.data_page_list[list_idx] if page.page_no == best_ent.pgid: page_found = True #we found the right data page, so start the file search search_start = page.blk_itab[best_ent.okey] * 0x100 #In the following lines: # The value d0 is to skip over the datapage header if we're in the first block # # The multiple of 0x4000 selects the data offset that goes with list_idx # since the parsed data-page list is in the same order as found in the file. # # Because mefs.data_page_list doesn't include the allocation table page, we +1 # to the index before multiplying. The result is a set of offsets into the MFS data # bounding the file search ## search_off = 0x4000 * (list_idx+1) + (0xd0 if search_start == 0 else search_start) me_file.seek(mfs_file_offset + search_off) data = read_me_fs_file(best_ent.fno, best_ent.filelen, me_file, log_file) if data: return [best_ent.state, data] return None if __name__ == "__main__": with open("image.bin", "rb") as spi_image_file: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) #Dump the allocation table mefs.allocation_table.debug_print() print("") print("looking up the first file (%s):" % mefs.allocation_table.entry_list[0].identifier) result_tuple = get_mfs_file(mefs, spi_image_file, MFS_OFFSET, b'UKS') if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple))

 b'UKS' len=0x0004 [pg=0x01 k=0x00 s=0x00] ty=0x0a st=0xdc, fg=0xf0 b'LRT' len=0x01c0 [pg=0x01 k=0x00 s=0x01] ty=0x0e st=0xdc, fg=0xf0 b'MIA' len=0x0003 [pg=0x01 k=0x02 s=0x01] ty=0x0e st=0xdc, fg=0xf0 b'BLV' len=0x0004 [pg=0x01 k=0x02 s=0x02] ty=0x0a st=0xdc, fg=0xf0 b'SDV' len=0x00ff [pg=0x01 k=0x02 s=0x03] ty=0x0a st=0xdc, fg=0xf0 b'ICP' len=0x0042 [pg=0x01 k=0x03 s=0x01] ty=0x0e st=0xdc, fg=0xf0 b'\x00NP' len=0x0001 [pg=0x01 k=0x04 s=0x01] ty=0x0e st=0xdc, fg=0xf0 b'PPN' len=0x0042 [pg=0x01 k=0x04 s=0x02] ty=0x0e st=0xdc, fg=0xf0 b'SCO' len=0x00af [pg=0x01 k=0x04 s=0x03] ty=0x0a st=0xdc, fg=0xf0 b'PCO' len=0x0a24 [pg=0x01 k=0x05 s=0x01] ty=0x0a st=0xdc, fg=0xf0 b'GFC' len=0x0004 [pg=0x01 k=0x07 s=0x01] ty=0x18 st=0xdc, fg=0xf0 b'YHP' len=0x06fe [pg=0x01 k=0x07 s=0x02] ty=0x0a st=0xdc, fg=0xf0 b'FCP' len=0x0002 [pg=0x01 k=0x09 s=0x01] ty=0x0b st=0xdc, fg=0xf0 b'PPR' len=0x0001 [pg=0x01 k=0x09 s=0x02] ty=0x0b st=0xdc, fg=0xf0 b'TCM' len=0x0005 [pg=0x01 k=0x09 s=0x03] ty=0x10 st=0xdc, fg=0xf0 b'BHM' len=0x0004 [pg=0x01 k=0x09 s=0x04] ty=0x10 st=0xdc, fg=0xf0 b'GCN' len=0x0018 [pg=0x01 k=0x09 s=0x05] ty=0x0e st=0xdc, fg=0xf0 b'CSM' len=0x000f [pg=0x01 k=0x0a s=0x00] ty=0x0e st=0xdc, fg=0xf0 b'\x00HS' len=0x0127 [pg=0x01 k=0x0a s=0x01] ty=0x0e st=0xdc, fg=0xf0 b'\x01HS' len=0x0127 [pg=0x01 k=0x0b s=0x01] ty=0x0e st=0xdc, fg=0xf0 b'\x02HS' len=0x0127 [pg=0x01 k=0x0c s=0x01] ty=0x0e st=0xdc, fg=0xf0 b'\x03HS' len=0x0127 [pg=0x01 k=0x0d s=0x01] ty=0x0e st=0xdc, fg=0xf0 b'PCF' len=0x0010 [pg=0x01 k=0x0e s=0x01] ty=0x0e st=0xdc, fg=0xf0 ...

import sys #litte-endian integer readers def read_leuint8(file): data = file.read(1) return data[0] def read_leuint16(file): data = file.read(2) return data[0] | (data[1] << 8) def read_leuint24(file): data = file.read(3) return data[0] | (data[1] << 8) | (data[2] << 16) def read_leuint32(file): data = file.read(4) return data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24) class MEFileSystemFileMetadataStateMachine: """ MEFileSystemFileMetadata: Files in MFS have internal metadata entries. Each file begins with a metadata record, observed values are: 0xa# <bytes ahead until next metadata> <0x01> <next meta block num> <0x00> 0xb# <blocks ahead until next metadata> <0x01> <next meta block num> <0x00> 0x8# <bytes remaining in file (including this metadata)> where # -- is the file number within the block. This is the target for the fno field of the allocation table entry. <bytes ahead...> -- the number of bytes to move ahead to find the next metadata record (including this metadata). <next meta block...>-- the 0x100-byte block number where the next metadata record is to be found. This value should be looked up through the data-page's block-indirection table. <blocks ahead...> -- the number of 0x100 byte blocks to move ahead to find the next metadata record (including this metadata). <bytes remaining...>-- pretty straight-forward. So, 0x8# metadata are file-end records; 0xa# are short-range references; and 0xb# are longer range references. This metadata chain provides unamiguous file numbers within the blocks, and since they're put at the block start of any block that contains a file-start, it's easy to pick up from an allocation table reference. Note: 0x8# records don't point to the next metadata block, so we may have to consume file-end padding (0xff until the next multiple of 0x10) if we get an intermediate 0x8# metadata while searching for our target file. """ STATE_NEED_META = 0 STATE_NEED_SKIP_DATA = 1 STATE_NEED_FILE_DATA = 2 STATE_COMPLETE = 3 def __init__(self, file_no, file_len): self.file_no = file_no self.state = MEFileSystemFileMetadataStateMachine.STATE_NEED_META self.bytes_needed = 1 self.byte_offset = 0 self.cur_meta = bytearray(5) self.file_data = bytearray(file_len) self.file_filled = 0 self.found_fileno = False self.work_buf = self.cur_meta def is_complete(self): return self.state == self.STATE_COMPLETE def get_file_data(self): return self.file_data def get_bytes_needed(self): return self.bytes_needed #returns the number of bytes consumed def add_bytes(self, bytes, start_index, data_len=None, log_file = None): """ supplies data to satisfy the state-machine's need for data as reported via get_bytes_needed(). bytes -- the buffer containing the bytes to be fed in start_index -- the start location of the bytes within the buffer data_len -- number of bytes in the array, starting at start_index. if None, then len(bytes) - start_index is assumed """ #shuffling data from potentially multiple calls to fill the data request from the #state machine (get_bytes_needed) data_len = len(bytes) - start_index if data_len is None else data_len if data_len == 0: return 0 # nothing to do #take the min of what's available and what we need to_copy = data_len if data_len < self.bytes_needed else self.bytes_needed if self.work_buf: self.work_buf[self.byte_offset:(self.byte_offset+to_copy)] = bytes[start_index:(start_index+to_copy)] self.byte_offset = self.byte_offset + to_copy self.bytes_needed = self.bytes_needed - to_copy #if we don't have enough to process, return so they can feed more if self.bytes_needed > 0: return to_copy #we only make it this far once we've got the full bytes_needed data meta_type = self.cur_meta[0] & 0xf0 if self.state == self.STATE_NEED_META: if self.byte_offset == 1: if meta_type in [0xa0, 0xb0]: self.bytes_needed = 4 else: self.bytes_needed = 1 else: #Have we found the file number we seek yet? if self.found_fileno or (self.file_no == self.cur_meta[0] & 0x0f): self.found_fileno = True self.state = self.STATE_NEED_FILE_DATA self.work_buf = self.file_data self.byte_offset = self.file_filled else: self.state = self.STATE_NEED_SKIP_DATA self.work_buf = None self.byte_offset = None #determine the data required based on metadata type, and whether we're #skipping (so need to eat EOF padding on type 0x8# entries) or whether #we're copying out file data. if meta_type == 0x80: if self.state == self.STATE_NEED_SKIP_DATA: #if we're skipping a 0x8# entry, we need to eat EOF padding too padding = (0x10 - (self.cur_meta[1] & 0xf)) & 0xf self.bytes_needed = padding + self.cur_meta[1] - 2 #remove header len, too else: self.bytes_needed = self.cur_meta[1] - 2 #remove header len elif meta_type == 0xa0: self.bytes_needed = self.cur_meta[1] - 5 #remove header len elif meta_type == 0xb0: self.bytes_needed = self.cur_meta[1] * 0x100 - 5 #remove header len else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_NEED_SKIP_DATA: #recall: this is the state just *completed* self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 self.bytes_needed = 1 elif self.state == self.STATE_NEED_FILE_DATA: #recall: this is the state just *completed* self.file_filled = self.byte_offset self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 if meta_type == 0x80: #just completed a file-end record...we're done. self.bytes_needed = 0 self.state = self.STATE_COMPLETE elif meta_type in [0xa0, 0xb0]: self.bytes_needed = 1 else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_COMPLETE: #can't leave this state pass else: if log_file: log_file.write("Bad state-machine state: %d\n" % self.state) #recurse to consume as much as we can, for easier calling convention if to_copy < data_len: return to_copy + add_bytes(bytes, start_index+to_copy, data_len-to_copy) #else, return what we consumed return to_copy def read_me_fs_file(file_no, file_len, me_file, log_file = sys.stdout): sm = MEFileSystemFileMetadataStateMachine(file_no, file_len) while not sm.is_complete(): res = sm.add_bytes( bytes=me_file.read(sm.get_bytes_needed()), start_index=0, data_len=None, #shorthand for len(bytes)-start_index log_file=log_file) if not res: log_file.write("Aborting file read.\n") break return sm.get_file_data() class MEFileSystemDataHeader: """ Data Page Header: (Top of a 0x4000-byte data page in the MFS) 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| pgflags | 0x00| 0x00| 0x00| 0x00| mystery bits | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | freed_flags... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ 80 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 90 | block indirection table... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ c0 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ d0+| (file data follows) | +-----------------------------------------------------------------------------------------------+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. pgflags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. myst_bits -- TBD freed_flags-- each bit marks whether a written 0x10-byte chunk is no longer required. Bits exist for the header chunks, as well as the chunks for file data. lsb of the first byte corresponds to the first chunk of the header. 0=freed. blk_itab -- translates ATFileEntry's okey value into a block offset within the data page. offset = blk_itab[okey] * 0x100. This is the location from which to begin the search. Note that an offset of 0 must still respect that the page header consumes bytes [0x00, 0xd0) for the search. (filedata) -- File are padded with 0xff at the end. Note files seem to have at least 2 bytes more data than the specified length. Larger files have more. """ def __init__(self, page_no, flags, bytes_00000000, myst_bits, freed_flags, blk_itab): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.myst_bits = myst_bits self.freed_flags = freed_flags self.blk_itab = blk_itab def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemDataHeader %x:\n" % id(self)) if self.page_no != 0xff: log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" mystery bits: %s\n" % " ".join("%02x"%x for x in self.myst_bits)) log.write(" freed_flags: %s\n" % "".join("%02x"%x for x in self.freed_flags)) log.write(" block ind. tab.: [%s]\n" % " ".join("%02x"%x for x in self.blk_itab)) else: log.write(" (empty)\n") class MEFileSystemATEntry: """ Allocation Table Entry: 0 1 2 3 4 5 6 7 8 9 a +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 |state| flgs| identifier | type| filelen | pgid| okey| fno | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ state -- status of the entry: 0xdc=present; 0xc8=overwritten(i.e., there will be another entry below) flags -- has value 0xf0 in every example...so hard to tell. identifier -- 3-byte identifier...seem to be a preference for ASCII, but there are counterexamples type -- no idea...maybe permissions? Observed values: 0x00, 0x0a, 0x0c, 0x0d, 0x0e, 0x18, 0x1a filelen -- 16-bit, little endian. actual content seems to be a few bytes more...might just be pollution of the structures used to write the file. pgid -- the pgno for the MEFileSystemDataHeader that holds the data. The ATHeader is numbered 0, each data page seems to get sequentially numbered in my examples, though that could change with more use. 0xff indicates "not yet used / nonumber assigned" --- a fact that hints page numbers aren't guaranteed to be sequentially okey -- key for indexing the block_indirection table of the MEFileSystemDataHeader holding this file's content, to find the right 0x100 byte block from which to start the file search according to the 'fno' field. fno -- file number within the block-offset determined from okey. This is to be looked up using the metadata *within* the file data of the data pages. See MEFileSystemFileMetadataStateMachine above """ def __init__(self, state, flags, identifier, type, filelen, pgid, okey, fno): self.state = state self.flags = flags self.identifier = identifier self.type = type self.filelen = filelen self.pgid = pgid self.okey = okey self.fno = fno def debug_print(self, log = sys.stdout): log.write("%15s len=0x%04x [pg=0x%02x k=0x%02x f=0x%02x] ty=0x%02x st=0x%02x, fg=0x%02x" % (str(self.identifier), self.filelen, self.pgid, self.okey, self.fno, self.type, self.state, self.flags)) class MEFileSystemATHeader: """ Allocation Table Header: 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| flags | 0x00| 0x00| 0x00| 0x00|"MFS\0" |bitfields? | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | bitfields? | +-----+-----+-----+-----+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. flags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. "MFS\0" -- ASCIIZ signature for the MFS AT Header bitfields -- 64 bits of apparent bitfields """ max_files = int((0x4000 - 0x14) / 11) #page size, less the header, divided by file entry size def __init__(self, page_no, flags, bytes_00000000, sig, bitfields): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.sig_good = sig == b'MFS\x00' self.bitfields = bitfields def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemATHeader %x:\n" % id(self)) log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" sig good: %s\n" % str(self.sig_good)) log.write(" bitfields: %s\n" % " ".join("%02x"%x for x in self.bitfields)) class MEFileSystemAT: """ Allocation Table (a 0x4000-byte page in the MFS): 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | Allocation Table Header... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 |... | File Entry |FE...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ ...|... | +-----------------------------------------------------------------------------------+-----+-----+ 3fe0|... |File Ent...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 3ff0|... |Padding | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ """ def __init__(self, header, entry_list): self.header = header self.entry_list = entry_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemAT %x:\n" % id(self)) self.header.debug_print(log) log.write("File entries:\n") for ent in self.entry_list: if(ent): log.write(" ") ent.debug_print(log) log.write("\n") class MEFileSystem: """ MFS Partition: (Making the assumption the AT is the first page) +------------------------------------+ 0000 | Allocation Table | +------------------------------------+ 4000 | Data Page | +------------------------------------+ 8000 | Data Page | +------------------------------------+ c000 | Data Page | +------------------------------------+ ... |... | +------------------------------------+ """ def __init__(self, allocation_table, data_page_list): self.allocation_table = allocation_table self.data_page_list = data_page_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystem %x:\n" % id(self)) self.allocation_table.debug_print(log) for page in self.data_page_list: if(page): page.debug_print(log) else: log.write("(Not parsed)\n") def parse_me_fs_at_entry(me_file, log_file = sys.stdout): return MEFileSystemATEntry( state = read_leuint8(me_file), flags = read_leuint8(me_file), identifier = me_file.read(3), type = read_leuint8(me_file), filelen = read_leuint16(me_file), pgid = read_leuint8(me_file), okey = read_leuint8(me_file), fno = read_leuint8(me_file)) def parse_me_fs_at_header(me_file, log_file = sys.stdout): return MEFileSystemATHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), sig = me_file.read(4), bitfields = me_file.read(8)) def parse_me_fs_at(me_file, log_file = sys.stdout): hdr = parse_me_fs_at_header(me_file, log_file) entry_list = [None] * MEFileSystemATHeader.max_files for i in range(MEFileSystemATHeader.max_files): ent = parse_me_fs_at_entry(me_file, log_file) if ent.state == 0xff: break entry_list[i] = ent return MEFileSystemAT(hdr, entry_list) def parse_me_fs_data_header(me_file, log_file = sys.stdout): return MEFileSystemDataHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), myst_bits = me_file.read(0x8), freed_flags = me_file.read(0x80), blk_itab = me_file.read(0x40)) def parse_me_fs(length, me_file, log_file = sys.stdout): """ ARGS: length -- length in bytes of the ME partition holding the MFS/MFSB data me_file -- a file handle whose present position is the start of the MFS(B) partition log_file -- if there is diagnostic output, put it here RETURNS: an MEFileSystem instance populated with the allocation table and data-page headers """ total_pages = int(length/0x4000) start_offset = me_file.tell() #NOTE: I'm presuming the allocation table is the first page... #that might not be reliable... at = parse_me_fs_at(me_file, log_file) data_pages = [None] * (total_pages-1) for page in range(0, total_pages-1): me_file.seek(start_offset + 0x4000 * (page+1)) data_pages[page] = parse_me_fs_data_header(me_file, log_file) return MEFileSystem( allocation_table=at, data_page_list = data_pages) def get_mfs_file(mefs, me_file, mfs_file_offset, id, log_file = sys.stdout): """ Example of how to use the MEFileSystem structures to retrieve MFS and MFSB files. ARGS: mefs -- a MEFileSystem instance parsed from mfs_data me_file -- a filehandle containing the ME image mfs_file_offset -- the file offset within me_file where the MFS partition begins id -- a 3-byte byte array with the file identifier log -- if there is diagnostic output, put it here RETURNS: an array containing [state, The data from the corresponding file]. else None if the file identifier does not exist within the data. Example driver, given the known offset and size for a MFS partition: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) result_tuple = get_mfs_file(mefs, spi_image_file, MFS_OFFSET, b'UKS') if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple)) """ #Find the file identifer in the Allocation Table best_ent = None for ent in mefs.allocation_table.entry_list: if ent and ent.identifier == id: best_ent = ent if ent.state == 0xdc: break; # got a current entry log.write("Error: found an item w/ state %02x...continuing\n" % ent.state) #if found, lookup which data page matches the entry's pgid value if best_ent: page_found = False for list_idx in range(len(mefs.data_page_list)): page = mefs.data_page_list[list_idx] if page.page_no == best_ent.pgid: page_found = True #we found the right data page, so start the file search search_start = page.blk_itab[best_ent.okey] * 0x100 #In the following lines: # The value d0 is to skip over the datapage header if we're in the first block # # The multiple of 0x4000 selects the data offset that goes with list_idx # since the parsed data-page list is in the same order as found in the file. # # Because mefs.data_page_list doesn't include the allocation table page, we +1 # to the index before multiplying. The result is a set of offsets into the MFS data # bounding the file search ## search_off = 0x4000 * (list_idx+1) + (0xd0 if search_start == 0 else search_start) me_file.seek(mfs_file_offset + search_off) data = read_me_fs_file(best_ent.fno, best_ent.filelen, me_file, log_file) if data: return [best_ent.state, data] return None if __name__ == "__main__": with open("image.bin", "rb") as spi_image_file: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) #Dump the allocation table mefs.allocation_table.debug_print() print("")   first_file = mefs.allocation_table.entry_list[0].identifier print("looking up the first file (%s):" % first_file) result_tuple = get_mfs_file(mefs, spi_image_file, MFS_OFFSET, first_file) if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple))

 b'UKS' len=0x0004 [pg=0x01 k=0x00 f=0x00] ty=0x0a st=0xdc, fg=0xf0 b'LRT' len=0x01c0 [pg=0x01 k=0x00 f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'MIA' len=0x0003 [pg=0x01 k=0x02 f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'BLV' len=0x0004 [pg=0x01 k=0x02 f=0x02] ty=0x0a st=0xdc, fg=0xf0 b'SDV' len=0x00ff [pg=0x01 k=0x02 f=0x03] ty=0x0a st=0xdc, fg=0xf0 b'ICP' len=0x0042 [pg=0x01 k=0x03 f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'\x00NP' len=0x0001 [pg=0x01 k=0x04 f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'PPN' len=0x0042 [pg=0x01 k=0x04 f=0x02] ty=0x0e st=0xdc, fg=0xf0 b'SCO' len=0x00af [pg=0x01 k=0x04 f=0x03] ty=0x0a st=0xdc, fg=0xf0 b'PCO' len=0x0a24 [pg=0x01 k=0x05 f=0x01] ty=0x0a st=0xdc, fg=0xf0 b'GFC' len=0x0004 [pg=0x01 k=0x07 f=0x01] ty=0x18 st=0xdc, fg=0xf0 b'YHP' len=0x06fe [pg=0x01 k=0x07 f=0x02] ty=0x0a st=0xdc, fg=0xf0 b'FCP' len=0x0002 [pg=0x01 k=0x09 f=0x01] ty=0x0b st=0xdc, fg=0xf0 b'PPR' len=0x0001 [pg=0x01 k=0x09 f=0x02] ty=0x0b st=0xdc, fg=0xf0 b'TCM' len=0x0005 [pg=0x01 k=0x09 f=0x03] ty=0x10 st=0xdc, fg=0xf0 b'BHM' len=0x0004 [pg=0x01 k=0x09 f=0x04] ty=0x10 st=0xdc, fg=0xf0 b'GCN' len=0x0018 [pg=0x01 k=0x09 f=0x05] ty=0x0e st=0xdc, fg=0xf0 b'CSM' len=0x000f [pg=0x01 k=0x0a f=0x00] ty=0x0e st=0xdc, fg=0xf0 b'\x00HS' len=0x0127 [pg=0x01 k=0x0a f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'\x01HS' len=0x0127 [pg=0x01 k=0x0b f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'\x02HS' len=0x0127 [pg=0x01 k=0x0c f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'\x03HS' len=0x0127 [pg=0x01 k=0x0d f=0x01] ty=0x0e st=0xdc, fg=0xf0 b'PCF' len=0x0010 [pg=0x01 k=0x0e f=0x01] ty=0x0e st=0xdc, fg=0xf0 ...

added 7075 characters in body

Source Link

edited Apr 6, 2020 at 9:23

lockcmpxchg8b

EDIT: There do seem to be metadata in the data. E.g., when looking at the content of UKS, the copy in the SCA partition is clearly 0E f4 00 00 whereas in MFS, the data there is 80 06 0e f4 00 00, so it looks like there's leading metadata. (And

EDIT 2: Figured out the metadata. It gives a much better method for deterministically identifying files by number within a block than the get_file inmethod I devised without it. Updated the code below is wrong...it doesn't yet know about thiswith a description and a rather complicated state-machine for successfully processing the metadata.)

import sys #litte-endian integer readers def read_leuint8(file): data = file.read(1) return data[0] def read_leuint16(file): data = file.read(2) return data[0] | (data[1] << 8) def read_leuint24(file): data = file.read(3) return data[0] | (data[1] << 8) | (data[2] << 16) def read_leuint32(file): data = file.read(4) return data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24) class MEFileSystemFileMetadataStateMachine: """ MEFileSystemFileMetadata: Files in MFS have internal metadata entries. Each file begins with a metadata record, observed values are: 0xa# <bytes ahead until next metadata> <0x01> <next meta block num> <0x00> 0xb# <blocks ahead until next metadata> <0x01> <next meta block num> <0x00> 0x8# <bytes remaining in file (including this metadata)> where # -- is the file number within the block. This is the target for the fno field of the allocation table entry. <bytes ahead...> -- the number of bytes to move ahead to find the next metadata record (including this metadata). <next meta block...>-- the 0x100-byte block number where the next metadata record is to be found. This value should be looked up through the data-page's block-indirection table. <blocks ahead...> -- the number of 0x100 byte blocks to move ahead to find the next metadata record (including this metadata). <bytes remaining...>-- pretty straight-forward. So, 0x8# metadata are file-end records; 0xa# are short-range references; and 0xb# are longer range references. This metadata chain provides unamiguous file numbers within the blocks, and since they're put at the block start of any block that contains a file-start, it's easy to pick up from an allocation table reference. Note: 0x8# records don't point to the next metadata block, so we may have to consume file-end padding (0xff until the next multiple of 0x10) if we get an intermediate 0x8# metadata while searching for our target file. """ STATE_NEED_META = 0 STATE_NEED_SKIP_DATA = 1 STATE_NEED_FILE_DATA = 2 STATE_COMPLETE = 3 def __init__(self, file_no, file_len): self.file_no = file_no self.state = MEFileSystemFileMetadataStateMachine.STATE_NEED_META self.bytes_needed = 1 self.byte_offset = 0 self.cur_meta = bytearray(5) self.file_data = bytearray(file_len) self.file_filled = 0 self.found_fileno = False self.work_buf = self.cur_meta def is_complete(self): return self.state == self.STATE_COMPLETE def get_file_data(self): return self.file_data def get_bytes_needed(self): return self.bytes_needed #returns the number of bytes consumed def add_bytes(self, bytes, start_index, data_len=None, log_file = None): """ supplies data to satisfy the state-machine's need for data as reported via get_bytes_needed(). bytes -- the buffer containing the bytes to be fed in start_index -- the start location of the bytes within the buffer data_len -- number of bytes in the array, starting at start_index. if None, then len(bytes) - start_index is assumed """ #shuffling data from potentially multiple calls to fill the data request from the #state machine (get_bytes_needed) data_len = len(bytes) - start_index if data_len is None else data_len if data_len == 0: return 0 # nothing to do #take the min of what's available and what we need to_copy = data_len if data_len < self.bytes_needed else self.bytes_needed if self.work_buf: self.work_buf[self.byte_offset:(self.byte_offset+to_copy)] = bytes[start_index:(start_index+to_copy)] self.byte_offset = self.byte_offset + to_copy self.bytes_needed = self.bytes_needed - to_copy #if we don't have enough to process, return so they can feed more if self.bytes_needed > 0: return to_copy #we only make it this far once we've got the full bytes_needed data meta_type = self.cur_meta[0] & 0xf0 if self.state == self.STATE_NEED_META: if self.byte_offset == 1: if meta_type in [0xa0, 0xb0]: self.bytes_needed = 4 else: self.bytes_needed = 1 else: #Have we found the file number we seek yet? if self.found_fileno or (self.file_no == self.cur_meta[0] & 0x0f): self.found_fileno = True self.state = self.STATE_NEED_FILE_DATA self.work_buf = self.file_data self.byte_offset = self.file_filled else: self.state = self.STATE_NEED_SKIP_DATA self.work_buf = None self.byte_offset = None #determine the data required based on metadata type, and whether we're #skipping (so need to eat EOF padding on type 0x8# entries) or whether #we're copying out file data. if meta_type == 0x80: if self.state == self.STATE_NEED_SKIP_DATA: #if we're skipping a 0x8# entry, we need to eat EOF padding too padding = (0x10 - (self.cur_meta[1] & 0xf)) & 0xf self.bytes_needed = padding + self.cur_meta[1] - 2 #remove header len, too else: self.bytes_needed = self.cur_meta[1] - 2 #remove header len elif meta_type == 0xa0: self.bytes_needed = self.cur_meta[1] - 5 #remove header len elif meta_type == 0xb0: self.bytes_needed = self.cur_meta[1] * 0x100 - 5 #remove header len else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_NEED_SKIP_DATA: #recall: this is the state just *completed* self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 self.bytes_needed = 1 elif self.state == self.STATE_NEED_FILE_DATA: #recall: this is the state just *completed* self.file_filled = self.byte_offset self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 if meta_type == 0x80: #just completed a file-end record...we're done. self.bytes_needed = 0 self.state = self.STATE_COMPLETE elif meta_type in [0xa0, 0xb0]: self.bytes_needed = 1 else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_COMPLETE: #can't leave this state pass else: if log_file: log_file.write("Bad state-machine state: %d\n" % self.state) #recurse to consume as much as we can, for easier calling convention if to_copy < data_len: return to_copy + add_bytes(bytes, start_index+to_copy, data_len-to_copy) #else, return what we consumed return to_copy def read_me_fs_file(file_no, file_len, me_file, log_file = sys.stdout): sm = MEFileSystemFileMetadataStateMachine(file_no, file_len) while not sm.is_complete(): res = sm.add_bytes( bytes=me_file.read(sm.get_bytes_needed()), start_index=0, data_len=None, #shorthand for len(bytes)-start_index log_file=log_file) if not res: log_file.write("Aborting file read.\n") break return sm.get_file_data() class MEFileSystemDataHeader: """ Data Page Header: (Top of a 0x4000-byte data page in the MFS) 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| pgflags | 0x00| 0x00| 0x00| 0x00| mystery bits | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | freed_flags... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ 80 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 90 | block indirection table... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ c0 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ d0+| (file data follows) | +-----------------------------------------------------------------------------------------------+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. pgflags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. myst_bits -- TBD freed_flags-- each bit marks whether a written 0x10-byte chunk is no longer required. Bits exist for the header chunks, as well as the chunks for file data. lsb of the first byte corresponds to the first chunk of the header. 0=freed. blk_itab -- translates ATFileEntry's okey value into a block offset within the data page. offset = blk_itab[okey] * 0x100. This is the location from which to begin the search. Note that an offset of 0 must still respect that the page header consumes bytes [0x00, 0xd0) for the search. (filedata) -- FilesFile are padded with 0xff at the end. Note files seem to have at least 2 bytes more data than the specified length. Larger files have more. """ def __init__(self, page_no, flags, bytes_00000000, myst_bits, freed_flags, blk_itab): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.myst_bits = myst_bits self.freed_flags = freed_flags self.blk_itab = blk_itab def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemDataHeader %x:\n" % id(self)) if self.page_no != 0xff: log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" mystery bits: %s\n" % " ".join("%02x"%x for x in self.myst_bits)) log.write(" freed_flags: %s\n" % "".join("%02x"%x for x in self.freed_flags)) log.write(" block ind. tab.: [%s]\n" % " ".join("%02x"%x for x in self.blk_itab)) else: log.write(" (empty)\n") class MEFileSystemATEntry: """ Allocation Table Entry: 0 1 2 3 4 5 6 7 8 9 a +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 |state| flgs| identifier | type| filelen |pgidx|| pgid| okey| skip|fno | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ state -- status of the entry: 0xdc=present; 0xc8=overwritten(i.e., there will be another entry below) flags -- has value 0xf0 in every example...so hard to tell. identifier -- 3-byte identifier...seem to be a preference for ASCII, but there are counterexamples type -- no idea...maybe permissions? Observed values: 0x00, 0x0a, 0x0c, 0x0d, 0x0e, 0x18, 0x1a filelen -- 16-bit, little endian. actual content seems to be a few bytes more...might just be pollution of the structures used to write the file. pgidxpgid  -- the pgno for the MEFileSystemDataHeader that holds the data. The ATHeader is numbered 0, each data page seems to get sequentially numbered in my examples, though that could change with more use. 0xff indicates "not yet used / nonumber assigned" --- a fact that hints page numbers aren't guaranteed to be sequentially okey -- key for indexing the block_indirection table of the MEFileSystemDataHeader holding this file's content, to find the right 0x100 byte block from which to start the file search according to the 'skip''fno' field. skipfno  -- perhaps the most amusing part of this format: starting  file number fromwithin the block-offset determined from okey, gives how.   many file end-padding sequences (sequences ofThis 0xff)is to  be skiplooked toup findusing the target file. Effectively,metadata "file*within* countthe   within the block". Notefile thatdata 0of isthe adata specialpages. case: whenSee   the file starts exactly on the blockMEFileSystemFileMetadataStateMachine bounary.above """ def __init__(self, state, flags, identifier, type, filelen, pgidxpgid, okey, skipfno): self.state = state self.flags = flags self.identifier = identifier self.type = type self.filelen = filelen self.pgidxpgid = pgidxpgid self.okey = okey self.skipfno = skipfno def debug_print(self, log = sys.stdout): log.write("%15s len=0x%04x [pg=0x%02x k=0x%02x s=0x%02x]f=0x%02x] ty=0x%02x st=0x%02x, fg=0x%02x" % (str(self.identifier), self.filelen, self.pgidxpgid, self.okey, self.skipfno, self.type, self.state, self.flags)) class MEFileSystemATHeader: """ Allocation Table Header: 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| flags | 0x00| 0x00| 0x00| 0x00|"MFS\0" |bitfields? | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | bitfields? | +-----+-----+-----+-----+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. flags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. "MFS\0" -- ASCIIZ signature for the MFS AT Header bitfields -- 64 bits of apparent bitfields """ max_files = int((0x4000 - 0x14) / 11) #page size, less the header, divided by file entry size def __init__(self, page_no, flags, bytes_00000000, sig, bitfields): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.sig_good = sig == b'MFS\x00' self.bitfields = bitfields def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemATHeader %x:\n" % id(self)) log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" sig good: %s\n" % str(self.sig_good)) log.write(" bitfields: %s\n" % " ".join("%02x"%x for x in self.bitfields)) class MEFileSystemAT: """ Allocation Table (a 0x4000-byte page in the MFS): 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | Allocation Table Header... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 |... | File Entry |FE...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ ...|... | +-----------------------------------------------------------------------------------+-----+-----+ 3fe0|... |File Ent...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 3ff0|... |Padding | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ """ def __init__(self, header, entry_list): self.header = header self.entry_list = entry_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemAT %x:\n" % id(self)) self.header.debug_print(log) log.write("File entries:\n") for ent in self.entry_list: if(ent): log.write(" ") ent.debug_print(log) log.write("\n") class MEFileSystem: """ MFS Partition: (Making the assumption the AT is the first page) +------------------------------------+ 0000 | Allocation Table | +------------------------------------+ 4000 | Data Page | +------------------------------------+ 8000 | Data Page | +------------------------------------+ c000 | Data Page | +------------------------------------+ ... |... | +------------------------------------+ """ def __init__(self, allocation_table, data_page_list): self.allocation_table = allocation_table self.data_page_list = data_page_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystem %x:\n" % id(self)) self.allocation_table.debug_print(log) for page in self.data_page_list: if(page): page.debug_print(log) else: log.write("(Not parsed)\n") def parse_me_fs_at_entry(me_file, log_file = sys.stdout): return MEFileSystemATEntry( state = read_leuint8(me_file), flags = read_leuint8(me_file), identifier = me_file.read(3), type = read_leuint8(me_file), filelen = read_leuint16(me_file), pgidxpgid = read_leuint8(me_file), okey = read_leuint8(me_file), skipfno = read_leuint8(me_file)) def parse_me_fs_at_header(me_file, log_file = sys.stdout): return MEFileSystemATHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), sig = me_file.read(4), bitfields = me_file.read(8)) def parse_me_fs_at(me_file, log_file = sys.stdout): hdr = parse_me_fs_at_header(me_file, log_file) entry_list = [None] * MEFileSystemATHeader.max_files for i in range(MEFileSystemATHeader.max_files): ent = parse_me_fs_at_entry(me_file, log_file) if ent.state == 0xff: break entry_list[i] = ent return MEFileSystemAT(hdr, entry_list) def parse_me_fs_data_header(me_file, log_file = sys.stdout): return MEFileSystemDataHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), myst_bits = me_file.read(0x8), freed_flags = me_file.read(0x80), blk_itab = me_file.read(0x40)) def parse_me_fs(length, me_file, log_file = sys.stdout): """ ARGS: length -- length in bytes of the ME partition holding the MFS/MFSB data me_file -- a file handle whose present position is the start of the MFS(B) partition log_file -- if there is diagnostic output, put it here RETURNS: an MEFileSystem instance populated with the allocation table and data-page headers """ total_pages = int(length/0x4000) start_offset = me_file.tell() #NOTE: I'm presuming the allocation table is the first page... #that might not be reliable... at = parse_me_fs_at(me_file, log_file) data_pages = [None] * (total_pages-1) for page in range(0, total_pages-1): me_file.seek(start_offset + 0x4000 * (page+1)) data_pages[page] = parse_me_fs_data_header(me_file, log_file) return MEFileSystem( allocation_table=at, data_page_list = data_pages) def get_mfs_file(mefs, mfs_datame_file, mfs_file_offset, id, loglog_file = sys.stdout): """ Example of how to use the MEFileSystem structures to retrieve MFS and MFSB files. ARGS: mefs -- a MEFileSystem instance parsed from mfs_data mfs_datame_file -- a filehandle containing the entireME MFSimage  partition, as a byte_array mfs_file_offset -- the file offset within me_file where the MFS partition begins id -- a 3-byte byte array with the file identifier log -- if there is diagnostic output, put it here RETURNS: an array containing [state, The data from the corresponding file]. else None if the file identifier does not exist within the data. Example driver, given the known offset and size for a MFS partition: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) spi_image_file.seek(MFS_OFFSET) mfs_data = spi_image_file.read(MFS_LENGTH) result_tuple = get_mfs_file(mefs, mfs_dataspi_image_file, MFS_OFFSET, b'UKS') if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple)) """ #Find the file identifer in the Allocation Table state = None filelen = None pgidx = None okey = None skip = None databest_ent = None for ent in mefs.allocation_table.entry_list: if ent and ent.identifier == id: state = ent.state filelen = ent.filelen pgidx = ent.pgidx okey = ent.okey skipbest_ent = ent.skip if ent.state == 0xdc: break; # got a current entry    log.write("Error: found an item w/ state %02x...continuing\n" % ent.state) #if found, lookup which data page matches the entry's pgidxpgid value if statebest_ent: page_found = False   for list_idx in range(len(mefs.data_page_list)): page = mefs.data_page_list[list_idx] if page.page_no == pgidxbest_ent.pgid: page_found = True #we found the right data page, so start the file search search_start = page.blk_itab[okey]blk_itab[best_ent.okey] * 0x100 #In the following lines: # The value d0 is to skip over the datapage header if we're in the first block # # The multiple of 0x4000 selects the data offset that goes with list_idx # since the parsed data-page list is in the same order as found in the file. # # Because mefs.data_page_list doesn't include the allocation table page, we +1 # to the index before multiplying. The result is a set of offsets into the MFS data # bounding the file search ## search_off = 0x4000 * (list_idx+1) + (0xd0 if search_start == 0 else search_start)  search_end = 0x4000 * (list_idx+1) + search_start + 0x100  skipped = 0 while search_off < search_end: #have we eaten enough files yet? if skipped == skip: return [state, mfs_data[search_off:me_file.seek(search_off+filelen)]] #else consume until we eat another end-marker while mfs_data[search_off] != 0xff: search_off = search_offmfs_file_offset + 1 while mfs_data[search_off] == 0xff:  search_off = search_off + 1)   skippeddata = skipped + 1   log.writeread_me_fs_file("Error: there weren't enough files to skip inbest_ent.fno, thebest_ent.filelen, targetme_file, block\n"log_file);     if not page_founddata: log.write("Error: corresponding data page doesn't exist (is this an new MFSB?)\n") return [best_ent.state, else:data] log.write("Error: no file record was found matching the identifier\n")  return None if __name__ == "__main__": with open("image.bin", "rb") as spi_image_file: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file)    spi_image_file#Dump the allocation table mefs.seekallocation_table.debug_print(MFS_OFFSET) mfs_dataprint("")  = spi_image_file.read print(MFS_LENGTH"looking up the first file (%s):" % mefs.allocation_table.entry_list[0].identifier) result_tuple = get_mfs_file(mefs, mfs_dataspi_image_file, MFS_OFFSET, b'UKS') if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple))

import sys #litte-endian integer readers def read_leuint8(file): data = file.read(1) return data[0] def read_leuint16(file): data = file.read(2) return data[0] | (data[1] << 8) def read_leuint24(file): data = file.read(3) return data[0] | (data[1] << 8) | (data[2] << 16) def read_leuint32(file): data = file.read(4) return data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24) class MEFileSystemDataHeader: """ Data Page Header: (Top of a 0x4000-byte data page in the MFS) 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| pgflags | 0x00| 0x00| 0x00| 0x00| mystery bits | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | freed_flags... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ 80 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 90 | block indirection table... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ c0 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ d0+| (file data follows) | +-----------------------------------------------------------------------------------------------+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. pgflags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. myst_bits -- TBD freed_flags-- each bit marks whether a written 0x10-byte chunk is no longer required. Bits exist for the header chunks, as well as the chunks for file data. lsb of the first byte corresponds to the first chunk of the header. 0=freed. blk_itab -- translates ATFileEntry's okey value into a block offset within the data page. offset = blk_itab[okey] * 0x100. This is the location from which to begin the search. Note that an offset of 0 must still respect that the page header consumes bytes [0x00, 0xd0) for the search. (filedata) -- Files are padded with 0xff at the end. Note files seem to have at least 2 bytes more data than the specified length. Larger files have more. """ def __init__(self, page_no, flags, bytes_00000000, myst_bits, freed_flags, blk_itab): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.myst_bits = myst_bits self.freed_flags = freed_flags self.blk_itab = blk_itab def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemDataHeader %x:\n" % id(self)) if self.page_no != 0xff: log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" mystery bits: %s\n" % " ".join("%02x"%x for x in self.myst_bits)) log.write(" freed_flags: %s\n" % "".join("%02x"%x for x in self.freed_flags)) log.write(" block ind. tab.: [%s]\n" % " ".join("%02x"%x for x in self.blk_itab)) else: log.write(" (empty)\n") class MEFileSystemATEntry: """ Allocation Table Entry: 0 1 2 3 4 5 6 7 8 9 a +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 |state| flgs| identifier | type| filelen |pgidx| okey| skip| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ state -- status of the entry: 0xdc=present; 0xc8=overwritten(i.e., there will be another entry below) flags -- has value 0xf0 in every example...so hard to tell. identifier -- 3-byte identifier...seem to be a preference for ASCII, but there are counterexamples type -- no idea...maybe permissions? Observed values: 0x00, 0x0a, 0x0c, 0x0d, 0x0e, 0x18, 0x1a filelen -- 16-bit, little endian. actual content seems to be a few bytes more...might just be pollution of the structures used to write the file. pgidx -- the pgno for the MEFileSystemDataHeader that holds the data. The ATHeader is numbered 0, each data page seems to get sequentially numbered in my examples, though that could change with more use. 0xff indicates "not yet used / nonumber assigned" --- a fact that hints page numbers aren't guaranteed to be sequentially okey -- key for indexing the block_indirection table of the MEFileSystemDataHeader holding this file's content, to find the right 0x100 byte block from which to start the file search according to the 'skip' field. skip -- perhaps the most amusing part of this format: starting  from the block-offset determined from okey, gives how   many file end-padding sequences (sequences of 0xff) to  skip to find the target file. Effectively, "file count   within the block". Note that 0 is a special case: when   the file starts exactly on the block bounary. """ def __init__(self, state, flags, identifier, type, filelen, pgidx, okey, skip): self.state = state self.flags = flags self.identifier = identifier self.type = type self.filelen = filelen self.pgidx = pgidx self.okey = okey self.skip = skip def debug_print(self, log = sys.stdout): log.write("%15s len=0x%04x [pg=0x%02x k=0x%02x s=0x%02x] ty=0x%02x st=0x%02x, fg=0x%02x" % (str(self.identifier), self.filelen, self.pgidx, self.okey, self.skip, self.type, self.state, self.flags)) class MEFileSystemATHeader: """ Allocation Table Header: 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| flags | 0x00| 0x00| 0x00| 0x00|"MFS\0" |bitfields? | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | bitfields? | +-----+-----+-----+-----+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. flags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. "MFS\0" -- ASCIIZ signature for the MFS AT Header bitfields -- 64 bits of apparent bitfields """ max_files = int((0x4000 - 0x14) / 11) #page size, less the header, divided by file entry size def __init__(self, page_no, flags, bytes_00000000, sig, bitfields): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.sig_good = sig == b'MFS\x00' self.bitfields = bitfields def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemATHeader %x:\n" % id(self)) log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" sig good: %s\n" % str(self.sig_good)) log.write(" bitfields: %s\n" % " ".join("%02x"%x for x in self.bitfields)) class MEFileSystemAT: """ Allocation Table (a 0x4000-byte page in the MFS): 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | Allocation Table Header... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 |... | File Entry |FE...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ ...|... | +-----------------------------------------------------------------------------------+-----+-----+ 3fe0|... |File Ent...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 3ff0|... |Padding | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ """ def __init__(self, header, entry_list): self.header = header self.entry_list = entry_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemAT %x:\n" % id(self)) self.header.debug_print(log) log.write("File entries:\n") for ent in self.entry_list: if(ent): log.write(" ") ent.debug_print(log) log.write("\n") class MEFileSystem: """ MFS Partition: (Making the assumption the AT is the first page) +------------------------------------+ 0000 | Allocation Table | +------------------------------------+ 4000 | Data Page | +------------------------------------+ 8000 | Data Page | +------------------------------------+ c000 | Data Page | +------------------------------------+ ... |... | +------------------------------------+ """ def __init__(self, allocation_table, data_page_list): self.allocation_table = allocation_table self.data_page_list = data_page_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystem %x:\n" % id(self)) self.allocation_table.debug_print(log) for page in self.data_page_list: if(page): page.debug_print(log) else: log.write("(Not parsed)\n") def parse_me_fs_at_entry(me_file, log_file = sys.stdout): return MEFileSystemATEntry( state = read_leuint8(me_file), flags = read_leuint8(me_file), identifier = me_file.read(3), type = read_leuint8(me_file), filelen = read_leuint16(me_file), pgidx = read_leuint8(me_file), okey = read_leuint8(me_file), skip = read_leuint8(me_file)) def parse_me_fs_at_header(me_file, log_file = sys.stdout): return MEFileSystemATHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), sig = me_file.read(4), bitfields = me_file.read(8)) def parse_me_fs_at(me_file, log_file = sys.stdout): hdr = parse_me_fs_at_header(me_file, log_file) entry_list = [None] * MEFileSystemATHeader.max_files for i in range(MEFileSystemATHeader.max_files): ent = parse_me_fs_at_entry(me_file, log_file) if ent.state == 0xff: break entry_list[i] = ent return MEFileSystemAT(hdr, entry_list) def parse_me_fs_data_header(me_file, log_file = sys.stdout): return MEFileSystemDataHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), myst_bits = me_file.read(0x8), freed_flags = me_file.read(0x80), blk_itab = me_file.read(0x40)) def parse_me_fs(length, me_file, log_file = sys.stdout): """ ARGS: length -- length in bytes of the ME partition holding the MFS/MFSB data me_file -- a file handle whose present position is the start of the MFS(B) partition log_file -- if there is diagnostic output, put it here RETURNS: an MEFileSystem instance populated with the allocation table and data-page headers """ total_pages = int(length/0x4000) start_offset = me_file.tell() #NOTE: I'm presuming the allocation table is the first page... #that might not be reliable... at = parse_me_fs_at(me_file, log_file) data_pages = [None] * (total_pages-1) for page in range(0, total_pages-1): me_file.seek(start_offset + 0x4000 * (page+1)) data_pages[page] = parse_me_fs_data_header(me_file, log_file) return MEFileSystem( allocation_table=at, data_page_list = data_pages) def get_mfs_file(mefs, mfs_data, id, log = sys.stdout): """ Example of how to use the MEFileSystem structures to retrieve MFS and MFSB files. ARGS: mefs -- a MEFileSystem instance parsed from mfs_data mfs_data -- the entire MFS partition, as a byte_array id -- a 3-byte byte array with the file identifier log -- if there is diagnostic output, put it here RETURNS: an array containing [state, The data from the corresponding file]. else None if the file identifier does not exist within the data. Example driver, given the known offset and size for a MFS partition: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) spi_image_file.seek(MFS_OFFSET) mfs_data = spi_image_file.read(MFS_LENGTH) result_tuple = get_mfs_file(mefs, mfs_data, b'UKS') if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple)) """ #Find the file identifer in the Allocation Table state = None filelen = None pgidx = None okey = None skip = None data = None for ent in mefs.allocation_table.entry_list: if ent and ent.identifier == id: state = ent.state filelen = ent.filelen pgidx = ent.pgidx okey = ent.okey skip = ent.skip if state == 0xdc: break; # got a current entry log.write("Error: found an item w/ state %02x...continuing\n" % state) #if found, lookup which data page matches the entry's pgidx value if state: page_found = False for list_idx in range(len(mefs.data_page_list)): page = mefs.data_page_list[list_idx] if page.page_no == pgidx: page_found = True #we found the right data page, so start the file search search_start = page.blk_itab[okey] * 0x100 #In the following lines: # The value d0 is to skip over the datapage header if we're in the first block # # The multiple of 0x4000 selects the data offset that goes with list_idx # since the parsed data-page list is in the same order as found in the file. # # Because mefs.data_page_list doesn't include the allocation table page, we +1 # to the index before multiplying. The result is a set of offsets into the MFS data # bounding the file search ## search_off = 0x4000 * (list_idx+1) + (0xd0 if search_start == 0 else search_start)  search_end = 0x4000 * (list_idx+1) + search_start + 0x100  skipped = 0 while search_off < search_end: #have we eaten enough files yet? if skipped == skip: return [state, mfs_data[search_off:(search_off+filelen)]] #else consume until we eat another end-marker while mfs_data[search_off] != 0xff: search_off = search_off + 1 while mfs_data[search_off] == 0xff:  search_off = search_off + 1   skipped = skipped + 1   log.write("Error: there weren't enough files to skip in the target block\n");     if not page_found: log.write("Error: corresponding data page doesn't exist (is this an new MFSB?)\n")  else: log.write("Error: no file record was found matching the identifier\n")  return None if __name__ == "__main__": with open("image.bin", "rb") as spi_image_file: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) spi_image_file.seek(MFS_OFFSET) mfs_data = spi_image_file.read(MFS_LENGTH) result_tuple = get_mfs_file(mefs, mfs_data, b'UKS') if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple))

EDIT 2: Figured out the metadata. It gives a much better method for deterministically identifying files by number within a block than the method I devised without it. Updated the code with a description and a rather complicated state-machine for successfully processing the metadata.

import sys #litte-endian integer readers def read_leuint8(file): data = file.read(1) return data[0] def read_leuint16(file): data = file.read(2) return data[0] | (data[1] << 8) def read_leuint24(file): data = file.read(3) return data[0] | (data[1] << 8) | (data[2] << 16) def read_leuint32(file): data = file.read(4) return data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24) class MEFileSystemFileMetadataStateMachine: """ MEFileSystemFileMetadata: Files in MFS have internal metadata entries. Each file begins with a metadata record, observed values are: 0xa# <bytes ahead until next metadata> <0x01> <next meta block num> <0x00> 0xb# <blocks ahead until next metadata> <0x01> <next meta block num> <0x00> 0x8# <bytes remaining in file (including this metadata)> where # -- is the file number within the block. This is the target for the fno field of the allocation table entry. <bytes ahead...> -- the number of bytes to move ahead to find the next metadata record (including this metadata). <next meta block...>-- the 0x100-byte block number where the next metadata record is to be found. This value should be looked up through the data-page's block-indirection table. <blocks ahead...> -- the number of 0x100 byte blocks to move ahead to find the next metadata record (including this metadata). <bytes remaining...>-- pretty straight-forward. So, 0x8# metadata are file-end records; 0xa# are short-range references; and 0xb# are longer range references. This metadata chain provides unamiguous file numbers within the blocks, and since they're put at the block start of any block that contains a file-start, it's easy to pick up from an allocation table reference. Note: 0x8# records don't point to the next metadata block, so we may have to consume file-end padding (0xff until the next multiple of 0x10) if we get an intermediate 0x8# metadata while searching for our target file. """ STATE_NEED_META = 0 STATE_NEED_SKIP_DATA = 1 STATE_NEED_FILE_DATA = 2 STATE_COMPLETE = 3 def __init__(self, file_no, file_len): self.file_no = file_no self.state = MEFileSystemFileMetadataStateMachine.STATE_NEED_META self.bytes_needed = 1 self.byte_offset = 0 self.cur_meta = bytearray(5) self.file_data = bytearray(file_len) self.file_filled = 0 self.found_fileno = False self.work_buf = self.cur_meta def is_complete(self): return self.state == self.STATE_COMPLETE def get_file_data(self): return self.file_data def get_bytes_needed(self): return self.bytes_needed #returns the number of bytes consumed def add_bytes(self, bytes, start_index, data_len=None, log_file = None): """ supplies data to satisfy the state-machine's need for data as reported via get_bytes_needed(). bytes -- the buffer containing the bytes to be fed in start_index -- the start location of the bytes within the buffer data_len -- number of bytes in the array, starting at start_index. if None, then len(bytes) - start_index is assumed """ #shuffling data from potentially multiple calls to fill the data request from the #state machine (get_bytes_needed) data_len = len(bytes) - start_index if data_len is None else data_len if data_len == 0: return 0 # nothing to do #take the min of what's available and what we need to_copy = data_len if data_len < self.bytes_needed else self.bytes_needed if self.work_buf: self.work_buf[self.byte_offset:(self.byte_offset+to_copy)] = bytes[start_index:(start_index+to_copy)] self.byte_offset = self.byte_offset + to_copy self.bytes_needed = self.bytes_needed - to_copy #if we don't have enough to process, return so they can feed more if self.bytes_needed > 0: return to_copy #we only make it this far once we've got the full bytes_needed data meta_type = self.cur_meta[0] & 0xf0 if self.state == self.STATE_NEED_META: if self.byte_offset == 1: if meta_type in [0xa0, 0xb0]: self.bytes_needed = 4 else: self.bytes_needed = 1 else: #Have we found the file number we seek yet? if self.found_fileno or (self.file_no == self.cur_meta[0] & 0x0f): self.found_fileno = True self.state = self.STATE_NEED_FILE_DATA self.work_buf = self.file_data self.byte_offset = self.file_filled else: self.state = self.STATE_NEED_SKIP_DATA self.work_buf = None self.byte_offset = None #determine the data required based on metadata type, and whether we're #skipping (so need to eat EOF padding on type 0x8# entries) or whether #we're copying out file data. if meta_type == 0x80: if self.state == self.STATE_NEED_SKIP_DATA: #if we're skipping a 0x8# entry, we need to eat EOF padding too padding = (0x10 - (self.cur_meta[1] & 0xf)) & 0xf self.bytes_needed = padding + self.cur_meta[1] - 2 #remove header len, too else: self.bytes_needed = self.cur_meta[1] - 2 #remove header len elif meta_type == 0xa0: self.bytes_needed = self.cur_meta[1] - 5 #remove header len elif meta_type == 0xb0: self.bytes_needed = self.cur_meta[1] * 0x100 - 5 #remove header len else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_NEED_SKIP_DATA: #recall: this is the state just *completed* self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 self.bytes_needed = 1 elif self.state == self.STATE_NEED_FILE_DATA: #recall: this is the state just *completed* self.file_filled = self.byte_offset self.state = self.STATE_NEED_META self.work_buf = self.cur_meta self.byte_offset = 0 if meta_type == 0x80: #just completed a file-end record...we're done. self.bytes_needed = 0 self.state = self.STATE_COMPLETE elif meta_type in [0xa0, 0xb0]: self.bytes_needed = 1 else: if log_file: log_file.write("That's not a metadata type I've seen before...: 0x%02x\n" % self.cur_meta[0]) return None elif self.state == self.STATE_COMPLETE: #can't leave this state pass else: if log_file: log_file.write("Bad state-machine state: %d\n" % self.state) #recurse to consume as much as we can, for easier calling convention if to_copy < data_len: return to_copy + add_bytes(bytes, start_index+to_copy, data_len-to_copy) #else, return what we consumed return to_copy def read_me_fs_file(file_no, file_len, me_file, log_file = sys.stdout): sm = MEFileSystemFileMetadataStateMachine(file_no, file_len) while not sm.is_complete(): res = sm.add_bytes( bytes=me_file.read(sm.get_bytes_needed()), start_index=0, data_len=None, #shorthand for len(bytes)-start_index log_file=log_file) if not res: log_file.write("Aborting file read.\n") break return sm.get_file_data() class MEFileSystemDataHeader: """ Data Page Header: (Top of a 0x4000-byte data page in the MFS) 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| pgflags | 0x00| 0x00| 0x00| 0x00| mystery bits | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | freed_flags... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ 80 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 90 | block indirection table... | +-----------------------------------------------------------------------------------------------+ ... | ... | +-----------------------------------------------------------------------------------------------+ c0 | ... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ d0+| (file data follows) | +-----------------------------------------------------------------------------------------------+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. pgflags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. myst_bits -- TBD freed_flags-- each bit marks whether a written 0x10-byte chunk is no longer required. Bits exist for the header chunks, as well as the chunks for file data. lsb of the first byte corresponds to the first chunk of the header. 0=freed. blk_itab -- translates ATFileEntry's okey value into a block offset within the data page. offset = blk_itab[okey] * 0x100. This is the location from which to begin the search. Note that an offset of 0 must still respect that the page header consumes bytes [0x00, 0xd0) for the search. (filedata) -- File are padded with 0xff at the end. Note files seem to have at least 2 bytes more data than the specified length. Larger files have more. """ def __init__(self, page_no, flags, bytes_00000000, myst_bits, freed_flags, blk_itab): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.myst_bits = myst_bits self.freed_flags = freed_flags self.blk_itab = blk_itab def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemDataHeader %x:\n" % id(self)) if self.page_no != 0xff: log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" mystery bits: %s\n" % " ".join("%02x"%x for x in self.myst_bits)) log.write(" freed_flags: %s\n" % "".join("%02x"%x for x in self.freed_flags)) log.write(" block ind. tab.: [%s]\n" % " ".join("%02x"%x for x in self.blk_itab)) else: log.write(" (empty)\n") class MEFileSystemATEntry: """ Allocation Table Entry: 0 1 2 3 4 5 6 7 8 9 a +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 |state| flgs| identifier | type| filelen | pgid| okey| fno | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ state -- status of the entry: 0xdc=present; 0xc8=overwritten(i.e., there will be another entry below) flags -- has value 0xf0 in every example...so hard to tell. identifier -- 3-byte identifier...seem to be a preference for ASCII, but there are counterexamples type -- no idea...maybe permissions? Observed values: 0x00, 0x0a, 0x0c, 0x0d, 0x0e, 0x18, 0x1a filelen -- 16-bit, little endian. actual content seems to be a few bytes more...might just be pollution of the structures used to write the file. pgid  -- the pgno for the MEFileSystemDataHeader that holds the data. The ATHeader is numbered 0, each data page seems to get sequentially numbered in my examples, though that could change with more use. 0xff indicates "not yet used / nonumber assigned" --- a fact that hints page numbers aren't guaranteed to be sequentially okey -- key for indexing the block_indirection table of the MEFileSystemDataHeader holding this file's content, to find the right 0x100 byte block from which to start the file search according to the 'fno' field. fno  -- file number within the block-offset determined from okey. This is to be looked up using the metadata *within* the file data of the data pages. See MEFileSystemFileMetadataStateMachine above """ def __init__(self, state, flags, identifier, type, filelen, pgid, okey, fno): self.state = state self.flags = flags self.identifier = identifier self.type = type self.filelen = filelen self.pgid = pgid self.okey = okey self.fno = fno def debug_print(self, log = sys.stdout): log.write("%15s len=0x%04x [pg=0x%02x k=0x%02x f=0x%02x] ty=0x%02x st=0x%02x, fg=0x%02x" % (str(self.identifier), self.filelen, self.pgid, self.okey, self.fno, self.type, self.state, self.flags)) class MEFileSystemATHeader: """ Allocation Table Header: 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | pgno| flags | 0x00| 0x00| 0x00| 0x00|"MFS\0" |bitfields? | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 | bitfields? | +-----+-----+-----+-----+ pgno -- page identifier within the MFS. All MFS page headers have this field, be they AT or Data. flags -- have seen 0x78 0xfe 0xff for AT header. Data pages have had this or 0x78 0xfc 0xff. (zeros) -- a useless value for flash...must be part of the signature of the header? Or reserved region for later version of the format. "MFS\0" -- ASCIIZ signature for the MFS AT Header bitfields -- 64 bits of apparent bitfields """ max_files = int((0x4000 - 0x14) / 11) #page size, less the header, divided by file entry size def __init__(self, page_no, flags, bytes_00000000, sig, bitfields): self.page_no = page_no self.flags = flags self.zeros_good = bytes_00000000 == b'\x00\x00\x00\x00' self.sig_good = sig == b'MFS\x00' self.bitfields = bitfields def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemATHeader %x:\n" % id(self)) log.write(" page no: 0x%x\n" % self.page_no) log.write(" flags: 0x%06x\n" % self.flags) log.write(" zeros good: %s\n" % str(self.zeros_good)) log.write(" sig good: %s\n" % str(self.sig_good)) log.write(" bitfields: %s\n" % " ".join("%02x"%x for x in self.bitfields)) class MEFileSystemAT: """ Allocation Table (a 0x4000-byte page in the MFS): 0 1 2 3 4 5 6 7 8 9 a b c d e f +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 00 | Allocation Table Header... | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 10 |... | File Entry |FE...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ ...|... | +-----------------------------------------------------------------------------------+-----+-----+ 3fe0|... |File Ent...| +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ 3ff0|... |Padding | +-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+ """ def __init__(self, header, entry_list): self.header = header self.entry_list = entry_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystemAT %x:\n" % id(self)) self.header.debug_print(log) log.write("File entries:\n") for ent in self.entry_list: if(ent): log.write(" ") ent.debug_print(log) log.write("\n") class MEFileSystem: """ MFS Partition: (Making the assumption the AT is the first page) +------------------------------------+ 0000 | Allocation Table | +------------------------------------+ 4000 | Data Page | +------------------------------------+ 8000 | Data Page | +------------------------------------+ c000 | Data Page | +------------------------------------+ ... |... | +------------------------------------+ """ def __init__(self, allocation_table, data_page_list): self.allocation_table = allocation_table self.data_page_list = data_page_list def debug_print(self, log = sys.stdout): log.write("Debug Print of MEFileSystem %x:\n" % id(self)) self.allocation_table.debug_print(log) for page in self.data_page_list: if(page): page.debug_print(log) else: log.write("(Not parsed)\n") def parse_me_fs_at_entry(me_file, log_file = sys.stdout): return MEFileSystemATEntry( state = read_leuint8(me_file), flags = read_leuint8(me_file), identifier = me_file.read(3), type = read_leuint8(me_file), filelen = read_leuint16(me_file), pgid = read_leuint8(me_file), okey = read_leuint8(me_file), fno = read_leuint8(me_file)) def parse_me_fs_at_header(me_file, log_file = sys.stdout): return MEFileSystemATHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), sig = me_file.read(4), bitfields = me_file.read(8)) def parse_me_fs_at(me_file, log_file = sys.stdout): hdr = parse_me_fs_at_header(me_file, log_file) entry_list = [None] * MEFileSystemATHeader.max_files for i in range(MEFileSystemATHeader.max_files): ent = parse_me_fs_at_entry(me_file, log_file) if ent.state == 0xff: break entry_list[i] = ent return MEFileSystemAT(hdr, entry_list) def parse_me_fs_data_header(me_file, log_file = sys.stdout): return MEFileSystemDataHeader( page_no = read_leuint8(me_file), flags = read_leuint24(me_file), bytes_00000000 = me_file.read(4), myst_bits = me_file.read(0x8), freed_flags = me_file.read(0x80), blk_itab = me_file.read(0x40)) def parse_me_fs(length, me_file, log_file = sys.stdout): """ ARGS: length -- length in bytes of the ME partition holding the MFS/MFSB data me_file -- a file handle whose present position is the start of the MFS(B) partition log_file -- if there is diagnostic output, put it here RETURNS: an MEFileSystem instance populated with the allocation table and data-page headers """ total_pages = int(length/0x4000) start_offset = me_file.tell() #NOTE: I'm presuming the allocation table is the first page... #that might not be reliable... at = parse_me_fs_at(me_file, log_file) data_pages = [None] * (total_pages-1) for page in range(0, total_pages-1): me_file.seek(start_offset + 0x4000 * (page+1)) data_pages[page] = parse_me_fs_data_header(me_file, log_file) return MEFileSystem( allocation_table=at, data_page_list = data_pages) def get_mfs_file(mefs, me_file, mfs_file_offset, id, log_file = sys.stdout): """ Example of how to use the MEFileSystem structures to retrieve MFS and MFSB files. ARGS: mefs -- a MEFileSystem instance parsed from mfs_data me_file -- a filehandle containing the ME image   mfs_file_offset -- the file offset within me_file where the MFS partition begins id -- a 3-byte byte array with the file identifier log -- if there is diagnostic output, put it here RETURNS: an array containing [state, The data from the corresponding file]. else None if the file identifier does not exist within the data. Example driver, given the known offset and size for a MFS partition: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file) result_tuple = get_mfs_file(mefs, spi_image_file, MFS_OFFSET, b'UKS') if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple)) """ #Find the file identifer in the Allocation Table best_ent = None for ent in mefs.allocation_table.entry_list: if ent and ent.identifier == id: best_ent = ent if ent.state == 0xdc: break; # got a current entry    log.write("Error: found an item w/ state %02x...continuing\n" % ent.state) #if found, lookup which data page matches the entry's pgid value if best_ent: page_found = False   for list_idx in range(len(mefs.data_page_list)): page = mefs.data_page_list[list_idx] if page.page_no == best_ent.pgid: page_found = True #we found the right data page, so start the file search search_start = page.blk_itab[best_ent.okey] * 0x100 #In the following lines: # The value d0 is to skip over the datapage header if we're in the first block # # The multiple of 0x4000 selects the data offset that goes with list_idx # since the parsed data-page list is in the same order as found in the file. # # Because mefs.data_page_list doesn't include the allocation table page, we +1 # to the index before multiplying. The result is a set of offsets into the MFS data # bounding the file search ## search_off = 0x4000 * (list_idx+1) + (0xd0 if search_start == 0 else search_start) me_file.seek(mfs_file_offset + search_off) data = read_me_fs_file(best_ent.fno, best_ent.filelen, me_file, log_file) if data: return [best_ent.state, data] return None if __name__ == "__main__": with open("image.bin", "rb") as spi_image_file: MFS_OFFSET = 0x64000 #typical location MFS_LENGTH = 0x40000 #typical size spi_image_file.seek(MFS_OFFSET) mefs = parse_me_fs(MFS_LENGTH, spi_image_file)    #Dump the allocation table mefs.allocation_table.debug_print() print("")   print("looking up the first file (%s):" % mefs.allocation_table.entry_list[0].identifier) result_tuple = get_mfs_file(mefs, spi_image_file, MFS_OFFSET, b'UKS') if result_tuple: print("State: %x, data: %s\n" % tuple(result_tuple))