using System.IO; using System.Text; namespace Agent.Windows.Tools; /// /// Detects whether a file is text or binary by reading only a small sample from the beginning. /// Uses BOM detection, UTF-16 validation, and UTF-8 strict decode — not NUL-byte-only detection /// (which would incorrectly reject valid UTF-16 text). /// public static class TextFileDetector { private const int SampleSize = 4096; /// /// Returns true if the file at appears to contain valid text /// (UTF-8, UTF-8 with BOM, UTF-16 LE, or UTF-16 BE). Reads at most bytes. /// public static bool IsText(string path) { using var stream = File.OpenRead(path); return IsText(stream); } /// /// Returns true if the stream's content appears to be valid text. /// Reads at most bytes from the current position. /// public static bool IsText(Stream stream) { byte[] buffer = new byte[SampleSize]; int read = stream.Read(buffer, 0, SampleSize); if (read == 0) return true; // Empty file is considered text. Span sample = buffer.AsSpan(0, read); // 1. Check BOM. if (sample.StartsWith(Encoding.UTF8.Preamble)) return TryDecodeUtf8(sample[3..]); if (sample.StartsWith(Encoding.Unicode.Preamble)) // UTF-16 LE return TryDecodeUtf16(sample[2..], bigEndian: false); if (sample.StartsWith(Encoding.BigEndianUnicode.Preamble)) // UTF-16 BE return TryDecodeUtf16(sample[2..], bigEndian: true); // 2. No BOM — try UTF-8 strict decode. return TryDecodeUtf8(sample); } private static bool TryDecodeUtf8(ReadOnlySpan data) { // Validate UTF-8 byte sequences manually to detect invalid bytes. return TryDecodeUtf8Core(data); } private static bool TryDecodeUtf8Core(ReadOnlySpan data) { // Validate UTF-8 byte sequences manually to detect invalid bytes. int i = 0; while (i < data.Length) { byte b = data[i]; if (b < 0x80) // ASCII { i++; } else if (b < 0xC2) // Invalid leading byte { return false; } else if (b < 0xE0) // 2-byte sequence { if (i + 1 >= data.Length) break; // Truncated at sample boundary — assume valid. if (!IsContinuationByte(data[i + 1])) return false; i += 2; } else if (b < 0xF0) // 3-byte sequence { if (i + 2 >= data.Length) break; if (!IsContinuationByte(data[i + 1]) || !IsContinuationByte(data[i + 2])) return false; // Check for overlong encodings and surrogate halves. if (b == 0xE0 && data[i + 1] < 0xA0) return false; // Overlong. if (b == 0xED && data[i + 1] >= 0xA0) return false; // Surrogate half. i += 3; } else if (b < 0xF5) // 4-byte sequence { if (i + 3 >= data.Length) break; if (!IsContinuationByte(data[i + 1]) || !IsContinuationByte(data[i + 2]) || !IsContinuationByte(data[i + 3])) return false; if (b == 0xF0 && data[i + 1] < 0x90) return false; // Overlong. if (b == 0xF4 && data[i + 1] >= 0x90) return false; // Out of range. i += 4; } else { return false; } } return true; } private static bool TryDecodeUtf16(ReadOnlySpan data, bool bigEndian) { // UTF-16: every code unit is 2 bytes. Validate that surrogates are properly paired. if (data.Length % 2 != 0) { // Odd byte count at sample boundary — check the pairs we have. } int i = 0; while (i + 1 < data.Length) { ushort codeUnit = bigEndian ? (ushort)((data[i] << 8) | data[i + 1]) : (ushort)((data[i + 1] << 8) | data[i]); if (codeUnit < 0xD800 || codeUnit > 0xDFFF) { // BMP character — valid. i += 2; } else if (codeUnit >= 0xD800 && codeUnit <= 0xDBFF) { // High surrogate — must be followed by a low surrogate. if (i + 3 >= data.Length) break; // Truncated at sample boundary — assume valid. ushort next = bigEndian ? (ushort)((data[i + 2] << 8) | data[i + 3]) : (ushort)((data[i + 3] << 8) | data[i + 2]); if (next < 0xDC00 || next > 0xDFFF) return false; i += 4; } else { // Low surrogate without preceding high surrogate — invalid. return false; } } return true; } private static bool IsContinuationByte(byte b) => b >= 0x80 && b <= 0xBF; }