Metadata overhaul + Phoenix overhaul WIP

bios_extract/src/phoenix.c

@@ -395,7 +395,7 @@ valid_signature:
 
 		memcpy(ModuleData, BIOSImage + Offset + Module->HeadLen,
 		       FragLength);
-		SetRemainder(Offset, Module->HeadLen + FragLength, FALSE);
+		//SetRemainder(Offset, Module->HeadLen + FragLength, FALSE);
 
 		Packed = FragLength;
 		FragOffset = le32toh(Module->NextFrag) & (BIOSLength - 1);
@@ -417,7 +417,7 @@ valid_signature:
 			Remain = BIOSLength - ((ModuleData + Packed) - BIOSImage);
 			memcpy(ModuleData + Packed, BIOSImage + FragOffset + 9,
 			       (Remain < FragLength) ? Remain : FragLength);
-			SetRemainder(FragOffset + 9, (Remain < FragLength) ? Remain : FragLength, FALSE);
+			//SetRemainder(FragOffset + 9, (Remain < FragLength) ? Remain : FragLength, FALSE);
 			Packed += FragLength;
 			FragOffset =
 			    le32toh(Fragment->NextFrag) & (BIOSLength - 1);
@@ -533,8 +533,8 @@ Uncompressed:
 
 	if (IsFragment)
 		free(ModuleData);
-	else
-		SetRemainder(Offset, Module->HeadLen + Packed, FALSE);
+	/*else
+		SetRemainder(Offset, Module->HeadLen + Packed, FALSE);*/
 
 	if (le16toh(Module->Offset) || le16toh(Module->Segment)) {
 		if (!Module->Compression)
@@ -1097,7 +1097,7 @@ PhoenixExtract(unsigned char *BIOSImage, int BIOSLength, int BIOSOffset,
 		if (!Buffer)
 			break;
 
-		SetRemainder(p - BIOSImage, sizeof(struct PhoenixBCD6F1) + le32toh(BCD6F1->FragLength), FALSE);
+		//SetRemainder(p - BIOSImage, sizeof(struct PhoenixBCD6F1) + le32toh(BCD6F1->FragLength), FALSE);
 
 		p += sizeof(struct PhoenixBCD6F1);
 		if (phx.compression == 0)
@@ -1138,7 +1138,7 @@ PhoenixExtract(unsigned char *BIOSImage, int BIOSLength, int BIOSOffset,
 		write(fd, p, Length);
 		close(fd);
 
-		SetRemainder(p - BIOSImage, Length, FALSE);
+		//SetRemainder(p - BIOSImage, Length, FALSE);
 		p += Length;
 	}
 
@@ -1170,7 +1170,7 @@ PhoenixExtract(unsigned char *BIOSImage, int BIOSLength, int BIOSOffset,
 		write(fd, p, Length);
 		close(fd);
 
-		SetRemainder(p - BIOSImage, Length, FALSE);
+		//SetRemainder(p - BIOSImage, Length, FALSE);
 		p += Length;
 	}
 
@@ -1215,7 +1215,7 @@ PhoenixExtract(unsigned char *BIOSImage, int BIOSLength, int BIOSOffset,
 		write(fd, p, Length);
 		close(fd);
 
-		SetRemainder(p - BIOSImage, Length, FALSE);
+		//SetRemainder(p - BIOSImage, Length, FALSE);
 		p += Length;
 	}
 
@@ -1249,7 +1249,7 @@ PhoenixExtract(unsigned char *BIOSImage, int BIOSLength, int BIOSOffset,
 		write(fd, p, Length);
 		close(fd);
 
-		SetRemainder(p - BIOSImage, Length, FALSE);
+		//SetRemainder(p - BIOSImage, Length, FALSE);
 		p += Length;
 	}
 
@@ -1279,11 +1279,12 @@ PhoenixExtract(unsigned char *BIOSImage, int BIOSLength, int BIOSOffset,
 		       Offset, Length);
 		write(fd, BIOSImage + Offset, Length);
 		close(fd);
-		SetRemainder(Offset, Length, FALSE);
+		//SetRemainder(Offset, Length, FALSE);
 	}
 
-	/* Manually flag BCPSEGMENT data as remaining, just in case */
-	SetRemainder(BCPSegmentOffset, ((unsigned char *)ID) - (BIOSImage + BCPSegmentOffset), TRUE);
+	/* Manually flag all data as remaining, which is required for register table
+	   and 4.0x sign-on analysis, as those use ROM-relative segments and offsets. */
+	SetRemainder(0, BIOSLength, TRUE);
 
 	return TRUE;
 }

biostools/__main__.py

@@ -304,7 +304,7 @@ def analyze_files(formatter, scan_base, file_analyzers, scan_dir_path, scan_file
 	# Sort file names for better predictability. The key= function forces
 	# "original.tm1" to be combined after "original.tmp" for if the Award
 	# identification data spans across both files (AOpen AX6B(+) R2.00)
-	if len(scan_file_names) > 1:		
+	if len(scan_file_names) > 1:
 		scan_file_names.sort(key=lambda fn: (fn == 'original.tm1') and 'original.tmq' or fn)
 
 	# Read files into the cache.
@@ -316,7 +316,7 @@ def analyze_files(formatter, scan_base, file_analyzers, scan_dir_path, scan_file
 		# Write data to cache.
 		if scan_file_name == ':header:':
 			header_data = file_data
-		elif combined:
+		elif combined and scan_file_name != ':combined:':
 			files_data[''] += file_data
 
 			# Add PCI option ROM IDs extracted from AMI BIOSes by bios_extract, since the ROM might not
@@ -360,7 +360,8 @@ def analyze_files(formatter, scan_base, file_analyzers, scan_dir_path, scan_file
 			file_data = util.read_complement(scan_file_path)
 
 		# Check for an analyzer which can handle this file.
-		bonus_analyzer_addons = bonus_analyzer_oroms = None
+		analyzer_file_path = combined and scan_dir_path or scan_file_path
+		bonus_analyzer_metadata = bonus_analyzer_oroms = None
 		file_analyzer = None
 		strings = None
 		for analyzer in file_analyzers:
@@ -370,7 +371,7 @@ def analyze_files(formatter, scan_base, file_analyzers, scan_dir_path, scan_file
 
 			# Check if the analyzer can handle this file.
 			try:
-				analyzer_result = analyzer.can_handle(file_data, header_data)
+				analyzer_result = analyzer.can_handle(analyzer_file_path, file_data, header_data)
 			except:
 				# Log an error.
 				util.log_traceback('searching for analyzers for', os.path.join(scan_dir_path, scan_file_name))
@@ -378,9 +379,9 @@ def analyze_files(formatter, scan_base, file_analyzers, scan_dir_path, scan_file
 
 			# Move on if the analyzer responded negatively.
 			if not analyzer_result:
-				# Extract add-ons and option ROMs from the bonus analyzer.
-				if bonus_analyzer_addons == None:
-					bonus_analyzer_addons = analyzer.addons
+				# Extract metadata and option ROMs from the bonus analyzer.
+				if bonus_analyzer_metadata == None:
+					bonus_analyzer_metadata = analyzer.metadata
 					bonus_analyzer_oroms = analyzer.oroms
 				continue
 
@@ -419,15 +420,15 @@ def analyze_files(formatter, scan_base, file_analyzers, scan_dir_path, scan_file
 		if not file_analyzer:
 			# Treat this as a standalone PCI option ROM file if BonusAnalyzer found any.
 			if bonus_analyzer_oroms:
-				bonus_analyzer_addons = []
+				bonus_analyzer_metadata = []
 				file_analyzer = file_analyzers[0]
 			else:
 				# Move on to the next file if nothing else.
 				continue
 
-		# Add interleaved flag to add-ons.
+		# Add interleaved flag to metadata.
 		if type(combined) == str:
-			bonus_analyzer_addons.append(combined)
+			bonus_analyzer_metadata.append(('ROM', combined))
 
 		# Clean up the file path.
 		scan_file_path_full = os.path.join(scan_dir_path, scan_file_name)
@@ -456,9 +457,9 @@ def analyze_files(formatter, scan_base, file_analyzers, scan_dir_path, scan_file
 		if slash_index == 1 and scan_file_path[0] == '0':
 			scan_file_path = scan_file_path[2:]
 
-		# De-duplicate and sort add-ons and option ROMs.
-		addons = list(set(addon.strip() for addon in (analyzer.addons + bonus_analyzer_addons)))
-		addons.sort()
+		# De-duplicate and sort metadata and option ROMs.
+		metadata = list(set('[{0}] {1}'.format(key, value).strip() for key, value in (analyzer.metadata + bonus_analyzer_metadata)))
+		metadata.sort()
 		oroms = list(set(combined_oroms + analyzer.oroms + bonus_analyzer_oroms))
 		oroms.sort()
 
@@ -525,7 +526,7 @@ def analyze_files(formatter, scan_base, file_analyzers, scan_dir_path, scan_file
 			file_analyzer.version,
 			formatter.split_if_required('\n', file_analyzer.string),
 			formatter.split_if_required('\n', file_analyzer.signon),
-			formatter.join_if_required(' ', addons),
+			formatter.join_if_required('\n', metadata),
 			formatter.join_if_required('\n', oroms),
 		]]
 
@@ -614,7 +615,7 @@ def analyze(dir_path, formatter_args, options):
 
 	# Begin output.
 	formatter.begin()
-	formatter.output_headers(['File', 'Vendor', 'Version', 'String', 'Sign-on', 'Add-ons', 'ROMs'], options.get('headers'))
+	formatter.output_headers(['File', 'Vendor', 'Version', 'String', 'Sign-on', 'Metadata', 'ROMs'], options.get('headers'))
 
 	# Remove any trailing slash from the root path, as the output path cleanup
 	# functions rely on it not being present.

biostools/formatters.py

@@ -97,6 +97,7 @@ class XSVFormatter(Formatter):
 		self.array = False
 
 		self.delimiter = delimiter
+		self.query = True
 
 		if self.options.get('hyperlink'):
 			# Get the localized HYPERLINK formula name if specified.
@@ -116,8 +117,11 @@ class XSVFormatter(Formatter):
 				link_prefix = '=' + self.hyperlink + '(""'
 				link_suffix = '""' + ';' + '""\U0001F53D"")' # down arrow emoji
 
-				# Build and output the final link, accounting for Excel's column size limit.
-				link = link_prefix + link_url[:256 - len(link_prefix) - len(link_suffix)] + link_suffix
+				# Build and output the final link, accounting for Excel's column size limit in connection (not query) mode.
+				if self.query:
+					link = link_prefix + link_url + link_suffix
+				else:
+					link = link_prefix + link_url[:256 - len(link_prefix) - len(link_suffix)] + link_suffix
 				output += '"' + link + '"'
 			else:
 				output += '""'
@@ -127,8 +131,11 @@ class XSVFormatter(Formatter):
 			if output:
 				output += self.delimiter
 			output += '"'
-			# Account for Excel's column size limit and lack of linebreak support.
-			output += field.replace('\n', ' - ').replace('"', '""')[:256]
+			if self.query:
+				output += field.replace('"', '""')
+			else:
+				# Account for Excel's column size limit and lack of linebreak support in connection (not query) mode.
+				output += field.replace('\n', ' - ').replace('"', '""')[:256]
 			output += '"'
 
 		# Add linebreak.

biostools/util.py

@@ -20,6 +20,7 @@ from biostools.pciutil import *
 
 date_pattern_mmddyy = re.compile('''(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/(?P<year>[0-9]{2,4})''')
 number_pattern = re.compile('''[0-9]+''')
+ascii_backspace_pattern = re.compile(b'''[\\x00-\\xFF]\\x08''')
 
 digits = '0123456789'
 lowercase = 'abcdefghijklmnopqrstuvwxyz'
@@ -191,11 +192,21 @@ def read_complement(file_path, file_header=None, max_size=16777216):
 			pass
 	return b''
 
-def read_string(data, terminator=b'\x00'):
+def read_string(data, terminator=b'\x00', ascii_backspace=True):
 	"""Read a terminated string (by NUL by default) from a bytes."""
+
+	# Trim to terminator.
 	terminator_index = data.find(terminator)
 	if terminator_index > -1:
 		data = data[:terminator_index]
+
+	# Look for ASCII backspaces and apply them accordingly.
+	if ascii_backspace:
+		replaced = 1
+		while replaced:
+			data, replaced = ascii_backspace_pattern.subn(b'', data)
+
+	# Decode as CP437.
 	return data.decode('cp437', 'ignore')
 
 def rmdirs(dir_path):