######################## BEGIN LICENSE BLOCK ########################
#
# Contributor(s):
# Jason Zavaglia
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA
######################### END LICENSE BLOCK #########################
from typing
import List, Union
from .charsetprober
import CharSetProber
from .enums
import ProbingState
class UTF1632Prober(CharSetProber):
"""
This
class simply looks
for occurrences of zero bytes,
and infers
whether the file
is UTF16
or UTF32 (low-endian
or big-endian)
For instance, files looking like ( \
0 \
0 \
0 [nonzero] )+
have a good probability to be UTF32BE. Files looking like ( \
0 [nonzero] )+
may be guessed to be UTF16BE,
and inversely
for little-endian varieties.
"""
# how many logical characters to scan before feeling confident of prediction
MIN_CHARS_FOR_DETECTION =
20
# a fixed constant ratio of expected zeros or non-zeros in modulo-position.
EXPECTED_RATIO =
0.
94
def __init__(self) ->
None:
super().__init__()
self.position =
0
self.zeros_at_mod = [
0] *
4
self.nonzeros_at_mod = [
0] *
4
self._state = ProbingState.DETECTING
self.quad = [
0,
0,
0,
0]
self.invalid_utf16be =
False
self.invalid_utf16le =
False
self.invalid_utf32be =
False
self.invalid_utf32le =
False
self.first_half_surrogate_pair_detected_16be =
False
self.first_half_surrogate_pair_detected_16le =
False
self.reset()
def reset(self) ->
None:
super().reset()
self.position =
0
self.zeros_at_mod = [
0] *
4
self.nonzeros_at_mod = [
0] *
4
self._state = ProbingState.DETECTING
self.invalid_utf16be =
False
self.invalid_utf16le =
False
self.invalid_utf32be =
False
self.invalid_utf32le =
False
self.first_half_surrogate_pair_detected_16be =
False
self.first_half_surrogate_pair_detected_16le =
False
self.quad = [
0,
0,
0,
0]
@property
def charset_name(self) -> str:
if self.is_likely_utf32be():
return "utf-32be"
if self.is_likely_utf32le():
return "utf-32le"
if self.is_likely_utf16be():
return "utf-16be"
if self.is_likely_utf16le():
return "utf-16le"
# default to something valid
return "utf-16"
@property
def language(self) -> str:
return ""
def approx_32bit_chars(self) -> float:
return max(
1.
0, self.position /
4.
0)
def approx_16bit_chars(self) -> float:
return max(
1.
0, self.position /
2.
0)
def is_likely_utf32be(self) -> bool:
approx_chars = self.approx_32bit_chars()
return approx_chars >= self.MIN_CHARS_FOR_DETECTION
and (
self.zeros_at_mod[
0] / approx_chars > self.EXPECTED_RATIO
and self.zeros_at_mod[
1] / approx_chars > self.EXPECTED_RATIO
and self.zeros_at_mod[
2] / approx_chars > self.EXPECTED_RATIO
and self.nonzeros_at_mod[
3] / approx_chars > self.EXPECTED_RATIO
and not self.invalid_utf32be
)
def is_likely_utf32le(self) -> bool:
approx_chars = self.approx_32bit_chars()
return approx_chars >= self.MIN_CHARS_FOR_DETECTION
and (
self.nonzeros_at_mod[
0] / approx_chars > self.EXPECTED_RATIO
and self.zeros_at_mod[
1] / approx_chars > self.EXPECTED_RATIO
and self.zeros_at_mod[
2] / approx_chars > self.EXPECTED_RATIO
and self.zeros_at_mod[
3] / approx_chars > self.EXPECTED_RATIO
and not self.invalid_utf32le
)
def is_likely_utf16be(self) -> bool:
approx_chars = self.approx_16bit_chars()
return approx_chars >= self.MIN_CHARS_FOR_DETECTION
and (
(self.nonzeros_at_mod[
1] + self.nonzeros_at_mod[
3]) / approx_chars
> self.EXPECTED_RATIO
and (self.zeros_at_mod[
0] + self.zeros_at_mod[
2]) / approx_chars
> self.EXPECTED_RATIO
and not self.invalid_utf16be
)
def is_likely_utf16le(self) -> bool:
approx_chars = self.approx_16bit_chars()
return approx_chars >= self.MIN_CHARS_FOR_DETECTION
and (
(self.nonzeros_at_mod[
0] + self.nonzeros_at_mod[
2]) / approx_chars
> self.EXPECTED_RATIO
and (self.zeros_at_mod[
1] + self.zeros_at_mod[
3]) / approx_chars
> self.EXPECTED_RATIO
and not self.invalid_utf16le
)
def validate_utf32_characters(self, quad: List[int]) ->
None:
"""
Validate
if the quad of bytes
is valid UTF-
32.
UTF-
32 is valid
in the range
0x00000000 -
0x0010FFFF
excluding
0x0000D800 -
0x0000DFFF
https://en.wikipedia.org/wiki/UTF-32
"""
if (
quad[
0] !=
0
or quad[
1] >
0x10
or (quad[
0] ==
0 and quad[
1] ==
0 and 0xD8 <= quad[
2] <=
0xDF)
):
self.invalid_utf32be =
True
if (
quad[
3] !=
0
or quad[
2] >
0x10
or (quad[
3] ==
0 and quad[
2] ==
0 and 0xD8 <= quad[
1] <=
0xDF)
):
self.invalid_utf32le =
True
def validate_utf16_characters(self, pair: List[int]) ->
None:
"""
Validate
if the pair of bytes
is valid UTF-
16.
UTF-
16 is valid
in the range
0x0000 -
0xFFFF excluding
0xD800 -
0xFFFF
with an exception
for surrogate pairs, which must be
in the range
0xD800-
0xDBFF followed by
0xDC00-
0xDFFF
https://en.wikipedia.org/wiki/UTF-16
"""
if not self.first_half_surrogate_pair_detected_16be:
if 0xD8 <= pair[
0] <=
0xDB:
self.first_half_surrogate_pair_detected_16be =
True
elif 0xDC <= pair[
0] <=
0xDF:
self.invalid_utf16be =
True
else:
if 0xDC <= pair[
0] <=
0xDF:
self.first_half_surrogate_pair_detected_16be =
False
else:
self.invalid_utf16be =
True
if not self.first_half_surrogate_pair_detected_16le:
if 0xD8 <= pair[
1] <=
0xDB:
self.first_half_surrogate_pair_detected_16le =
True
elif 0xDC <= pair[
1] <=
0xDF:
self.invalid_utf16le =
True
else:
if 0xDC <= pair[
1] <=
0xDF:
self.first_half_surrogate_pair_detected_16le =
False
else:
self.invalid_utf16le =
True
def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
for c
in byte_str:
mod4 = self.position %
4
self.quad[mod4] = c
if mod4 ==
3:
self.validate_utf32_characters(self.quad)
self.validate_utf16_characters(self.quad[
0:
2])
self.validate_utf16_characters(self.quad[
2:
4])
if c ==
0:
self.zeros_at_mod[mod4] +=
1
else:
self.nonzeros_at_mod[mod4] +=
1
self.position +=
1
return self.state
@property
def state(self) -> ProbingState:
if self._state
in {ProbingState.NOT_ME, ProbingState.FOUND_IT}:
# terminal, decided states
return self._state
if self.get_confidence() >
0.
80:
self._state = ProbingState.FOUND_IT
elif self.position >
4 *
1024:
# if we get to 4kb into the file, and we can't conclude it's UTF,
# let's give up
self._state = ProbingState.NOT_ME
return self._state
def get_confidence(self) -> float:
return (
0.
85
if (
self.is_likely_utf16le()
or self.is_likely_utf16be()
or self.is_likely_utf32le()
or self.is_likely_utf32be()
)
else 0.
00
)
