using System.IO;
using System.Text;
namespace Agent.Windows.Tools;
///
/// Detects whether a file is text or binary by reading only a small sample from the beginning.
/// Uses BOM detection, UTF-16 validation, and UTF-8 strict decode — not NUL-byte-only detection
/// (which would incorrectly reject valid UTF-16 text).
///
public static class TextFileDetector
{
private const int SampleSize = 4096;
///
/// Returns true if the file at appears to contain valid text
/// (UTF-8, UTF-8 with BOM, UTF-16 LE, or UTF-16 BE). Reads at most bytes.
///
public static bool IsText(string path)
{
using var stream = File.OpenRead(path);
return IsText(stream);
}
///
/// Returns true if the stream's content appears to be valid text.
/// Reads at most bytes from the current position.
///
public static bool IsText(Stream stream)
{
byte[] buffer = new byte[SampleSize];
int read = stream.Read(buffer, 0, SampleSize);
if (read == 0) return true; // Empty file is considered text.
Span sample = buffer.AsSpan(0, read);
// 1. Check BOM.
if (sample.StartsWith(Encoding.UTF8.Preamble))
return TryDecodeUtf8(sample[3..]);
if (sample.StartsWith(Encoding.Unicode.Preamble)) // UTF-16 LE
return TryDecodeUtf16(sample[2..], bigEndian: false);
if (sample.StartsWith(Encoding.BigEndianUnicode.Preamble)) // UTF-16 BE
return TryDecodeUtf16(sample[2..], bigEndian: true);
// 2. No BOM — try UTF-8 strict decode.
return TryDecodeUtf8(sample);
}
private static bool TryDecodeUtf8(ReadOnlySpan data)
{
// Validate UTF-8 byte sequences manually to detect invalid bytes.
return TryDecodeUtf8Core(data);
}
private static bool TryDecodeUtf8Core(ReadOnlySpan data)
{
// Validate UTF-8 byte sequences manually to detect invalid bytes.
int i = 0;
while (i < data.Length)
{
byte b = data[i];
if (b < 0x80) // ASCII
{
i++;
}
else if (b < 0xC2) // Invalid leading byte
{
return false;
}
else if (b < 0xE0) // 2-byte sequence
{
if (i + 1 >= data.Length) break; // Truncated at sample boundary — assume valid.
if (!IsContinuationByte(data[i + 1])) return false;
i += 2;
}
else if (b < 0xF0) // 3-byte sequence
{
if (i + 2 >= data.Length) break;
if (!IsContinuationByte(data[i + 1]) || !IsContinuationByte(data[i + 2])) return false;
// Check for overlong encodings and surrogate halves.
if (b == 0xE0 && data[i + 1] < 0xA0) return false; // Overlong.
if (b == 0xED && data[i + 1] >= 0xA0) return false; // Surrogate half.
i += 3;
}
else if (b < 0xF5) // 4-byte sequence
{
if (i + 3 >= data.Length) break;
if (!IsContinuationByte(data[i + 1]) || !IsContinuationByte(data[i + 2]) || !IsContinuationByte(data[i + 3])) return false;
if (b == 0xF0 && data[i + 1] < 0x90) return false; // Overlong.
if (b == 0xF4 && data[i + 1] >= 0x90) return false; // Out of range.
i += 4;
}
else
{
return false;
}
}
return true;
}
private static bool TryDecodeUtf16(ReadOnlySpan data, bool bigEndian)
{
// UTF-16: every code unit is 2 bytes. Validate that surrogates are properly paired.
if (data.Length % 2 != 0)
{
// Odd byte count at sample boundary — check the pairs we have.
}
int i = 0;
while (i + 1 < data.Length)
{
ushort codeUnit = bigEndian
? (ushort)((data[i] << 8) | data[i + 1])
: (ushort)((data[i + 1] << 8) | data[i]);
if (codeUnit < 0xD800 || codeUnit > 0xDFFF)
{
// BMP character — valid.
i += 2;
}
else if (codeUnit >= 0xD800 && codeUnit <= 0xDBFF)
{
// High surrogate — must be followed by a low surrogate.
if (i + 3 >= data.Length) break; // Truncated at sample boundary — assume valid.
ushort next = bigEndian
? (ushort)((data[i + 2] << 8) | data[i + 3])
: (ushort)((data[i + 3] << 8) | data[i + 2]);
if (next < 0xDC00 || next > 0xDFFF) return false;
i += 4;
}
else
{
// Low surrogate without preceding high surrogate — invalid.
return false;
}
}
return true;
}
private static bool IsContinuationByte(byte b) => b >= 0x80 && b <= 0xBF;
}