diff options
Diffstat (limited to 'src/Agent.Windows/Tools/TextFileDetector.cs')
| -rw-r--r-- | src/Agent.Windows/Tools/TextFileDetector.cs | 141 |
1 files changed, 141 insertions, 0 deletions
diff --git a/src/Agent.Windows/Tools/TextFileDetector.cs b/src/Agent.Windows/Tools/TextFileDetector.cs new file mode 100644 index 0000000..9a5778b --- /dev/null +++ b/src/Agent.Windows/Tools/TextFileDetector.cs @@ -0,0 +1,141 @@ +using System.IO; +using System.Text; + +namespace Agent.Windows.Tools; + +/// <summary> +/// Detects whether a file is text or binary by reading only a small sample from the beginning. +/// Uses BOM detection, UTF-16 validation, and UTF-8 strict decode — not NUL-byte-only detection +/// (which would incorrectly reject valid UTF-16 text). +/// </summary> +public static class TextFileDetector +{ + private const int SampleSize = 4096; + + /// <summary> + /// Returns true if the file at <paramref name="path"/> appears to contain valid text + /// (UTF-8, UTF-8 with BOM, UTF-16 LE, or UTF-16 BE). Reads at most <see cref="SampleSize"/> bytes. + /// </summary> + public static bool IsText(string path) + { + using var stream = File.OpenRead(path); + return IsText(stream); + } + + /// <summary> + /// Returns true if the stream's content appears to be valid text. + /// Reads at most <see cref="SampleSize"/> bytes from the current position. + /// </summary> + public static bool IsText(Stream stream) + { + byte[] buffer = new byte[SampleSize]; + int read = stream.Read(buffer, 0, SampleSize); + if (read == 0) return true; // Empty file is considered text. + + Span<byte> sample = buffer.AsSpan(0, read); + + // 1. Check BOM. + if (sample.StartsWith(Encoding.UTF8.Preamble)) + return TryDecodeUtf8(sample[3..]); + if (sample.StartsWith(Encoding.Unicode.Preamble)) // UTF-16 LE + return TryDecodeUtf16(sample[2..], bigEndian: false); + if (sample.StartsWith(Encoding.BigEndianUnicode.Preamble)) // UTF-16 BE + return TryDecodeUtf16(sample[2..], bigEndian: true); + + // 2. No BOM — try UTF-8 strict decode. + return TryDecodeUtf8(sample); + } + + private static bool TryDecodeUtf8(ReadOnlySpan<byte> data) + { + // Validate UTF-8 byte sequences manually to detect invalid bytes. + return TryDecodeUtf8Core(data); + } + + private static bool TryDecodeUtf8Core(ReadOnlySpan<byte> data) + { + // Validate UTF-8 byte sequences manually to detect invalid bytes. + int i = 0; + while (i < data.Length) + { + byte b = data[i]; + if (b < 0x80) // ASCII + { + i++; + } + else if (b < 0xC2) // Invalid leading byte + { + return false; + } + else if (b < 0xE0) // 2-byte sequence + { + if (i + 1 >= data.Length) break; // Truncated at sample boundary — assume valid. + if (!IsContinuationByte(data[i + 1])) return false; + i += 2; + } + else if (b < 0xF0) // 3-byte sequence + { + if (i + 2 >= data.Length) break; + if (!IsContinuationByte(data[i + 1]) || !IsContinuationByte(data[i + 2])) return false; + // Check for overlong encodings and surrogate halves. + if (b == 0xE0 && data[i + 1] < 0xA0) return false; // Overlong. + if (b == 0xED && data[i + 1] >= 0xA0) return false; // Surrogate half. + i += 3; + } + else if (b < 0xF5) // 4-byte sequence + { + if (i + 3 >= data.Length) break; + if (!IsContinuationByte(data[i + 1]) || !IsContinuationByte(data[i + 2]) || !IsContinuationByte(data[i + 3])) return false; + if (b == 0xF0 && data[i + 1] < 0x90) return false; // Overlong. + if (b == 0xF4 && data[i + 1] >= 0x90) return false; // Out of range. + i += 4; + } + else + { + return false; + } + } + return true; + } + + private static bool TryDecodeUtf16(ReadOnlySpan<byte> data, bool bigEndian) + { + // UTF-16: every code unit is 2 bytes. Validate that surrogates are properly paired. + if (data.Length % 2 != 0) + { + // Odd byte count at sample boundary — check the pairs we have. + } + + int i = 0; + while (i + 1 < data.Length) + { + ushort codeUnit = bigEndian + ? (ushort)((data[i] << 8) | data[i + 1]) + : (ushort)((data[i + 1] << 8) | data[i]); + + if (codeUnit < 0xD800 || codeUnit > 0xDFFF) + { + // BMP character — valid. + i += 2; + } + else if (codeUnit >= 0xD800 && codeUnit <= 0xDBFF) + { + // High surrogate — must be followed by a low surrogate. + if (i + 3 >= data.Length) break; // Truncated at sample boundary — assume valid. + ushort next = bigEndian + ? (ushort)((data[i + 2] << 8) | data[i + 3]) + : (ushort)((data[i + 3] << 8) | data[i + 2]); + if (next < 0xDC00 || next > 0xDFFF) return false; + i += 4; + } + else + { + // Low surrogate without preceding high surrogate — invalid. + return false; + } + } + return true; + } + + private static bool IsContinuationByte(byte b) => b >= 0x80 && b <= 0xBF; +}
\ No newline at end of file |