blob: 9a5778b38749663496170efd6235a2d88acd58f5 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
|
using System.IO;
using System.Text;
namespace Agent.Windows.Tools;
/// <summary>
/// Detects whether a file is text or binary by reading only a small sample from the beginning.
/// Uses BOM detection, UTF-16 validation, and UTF-8 strict decode — not NUL-byte-only detection
/// (which would incorrectly reject valid UTF-16 text).
/// </summary>
public static class TextFileDetector
{
private const int SampleSize = 4096;
/// <summary>
/// Returns true if the file at <paramref name="path"/> appears to contain valid text
/// (UTF-8, UTF-8 with BOM, UTF-16 LE, or UTF-16 BE). Reads at most <see cref="SampleSize"/> bytes.
/// </summary>
public static bool IsText(string path)
{
using var stream = File.OpenRead(path);
return IsText(stream);
}
/// <summary>
/// Returns true if the stream's content appears to be valid text.
/// Reads at most <see cref="SampleSize"/> bytes from the current position.
/// </summary>
public static bool IsText(Stream stream)
{
byte[] buffer = new byte[SampleSize];
int read = stream.Read(buffer, 0, SampleSize);
if (read == 0) return true; // Empty file is considered text.
Span<byte> sample = buffer.AsSpan(0, read);
// 1. Check BOM.
if (sample.StartsWith(Encoding.UTF8.Preamble))
return TryDecodeUtf8(sample[3..]);
if (sample.StartsWith(Encoding.Unicode.Preamble)) // UTF-16 LE
return TryDecodeUtf16(sample[2..], bigEndian: false);
if (sample.StartsWith(Encoding.BigEndianUnicode.Preamble)) // UTF-16 BE
return TryDecodeUtf16(sample[2..], bigEndian: true);
// 2. No BOM — try UTF-8 strict decode.
return TryDecodeUtf8(sample);
}
private static bool TryDecodeUtf8(ReadOnlySpan<byte> data)
{
// Validate UTF-8 byte sequences manually to detect invalid bytes.
return TryDecodeUtf8Core(data);
}
private static bool TryDecodeUtf8Core(ReadOnlySpan<byte> data)
{
// Validate UTF-8 byte sequences manually to detect invalid bytes.
int i = 0;
while (i < data.Length)
{
byte b = data[i];
if (b < 0x80) // ASCII
{
i++;
}
else if (b < 0xC2) // Invalid leading byte
{
return false;
}
else if (b < 0xE0) // 2-byte sequence
{
if (i + 1 >= data.Length) break; // Truncated at sample boundary — assume valid.
if (!IsContinuationByte(data[i + 1])) return false;
i += 2;
}
else if (b < 0xF0) // 3-byte sequence
{
if (i + 2 >= data.Length) break;
if (!IsContinuationByte(data[i + 1]) || !IsContinuationByte(data[i + 2])) return false;
// Check for overlong encodings and surrogate halves.
if (b == 0xE0 && data[i + 1] < 0xA0) return false; // Overlong.
if (b == 0xED && data[i + 1] >= 0xA0) return false; // Surrogate half.
i += 3;
}
else if (b < 0xF5) // 4-byte sequence
{
if (i + 3 >= data.Length) break;
if (!IsContinuationByte(data[i + 1]) || !IsContinuationByte(data[i + 2]) || !IsContinuationByte(data[i + 3])) return false;
if (b == 0xF0 && data[i + 1] < 0x90) return false; // Overlong.
if (b == 0xF4 && data[i + 1] >= 0x90) return false; // Out of range.
i += 4;
}
else
{
return false;
}
}
return true;
}
private static bool TryDecodeUtf16(ReadOnlySpan<byte> data, bool bigEndian)
{
// UTF-16: every code unit is 2 bytes. Validate that surrogates are properly paired.
if (data.Length % 2 != 0)
{
// Odd byte count at sample boundary — check the pairs we have.
}
int i = 0;
while (i + 1 < data.Length)
{
ushort codeUnit = bigEndian
? (ushort)((data[i] << 8) | data[i + 1])
: (ushort)((data[i + 1] << 8) | data[i]);
if (codeUnit < 0xD800 || codeUnit > 0xDFFF)
{
// BMP character — valid.
i += 2;
}
else if (codeUnit >= 0xD800 && codeUnit <= 0xDBFF)
{
// High surrogate — must be followed by a low surrogate.
if (i + 3 >= data.Length) break; // Truncated at sample boundary — assume valid.
ushort next = bigEndian
? (ushort)((data[i + 2] << 8) | data[i + 3])
: (ushort)((data[i + 3] << 8) | data[i + 2]);
if (next < 0xDC00 || next > 0xDFFF) return false;
i += 4;
}
else
{
// Low surrogate without preceding high surrogate — invalid.
return false;
}
}
return true;
}
private static bool IsContinuationByte(byte b) => b >= 0x80 && b <= 0xBF;
}
|
