Streamly.Internal.Unicode.Stream · streamly-core-0.2.2

Setup

0 declarations

To execute the code examples provided in this module in ghci, please run the following commands first.

Example1 expression

>>> :m

Example3 expressions

>>> import qualified Streamly.Data.Fold as Fold>>> import qualified Streamly.Data.Stream as Stream>>> import qualified Streamly.Unicode.Stream as Unicode

For APIs that have not been released yet.

Example2 expressions

>>> :set -XMagicHash>>> import qualified Streamly.Internal.Unicode.Stream as Unicode

Construction (Decoding)

1 declaration

valuedecodeLatin1 :: Monad m => Stream m Word8 -> Stream m Char

#

Decode a stream of bytes to Unicode characters by mapping each byte to a corresponding Unicode Char in 0-255 range.

UTF-8 Decoding

datadata CodingFailureMode

#

TransliterateCodingFailure ErrorOnCodingFailure DropOnCodingFailure

Instances1Show

Show CodingFailureModeDefined in streamly-core-0.2.2 · Streamly.Internal.Unicode.Stream

valuewriteCharUtf8' :: Monad m => Parser Word8 m Char

#

valueparseCharUtf8With :: Monad m => CodingFailureMode -> Parser Word8 m Char

#

valuedecodeUtf8 :: Monad m => Stream m Word8 -> Stream m Char

#

Decode a UTF-8 encoded bytestream to a stream of Unicode characters. Any invalid codepoint encountered is replaced with the unicode replacement character.

valuedecodeUtf8' :: Monad m => Stream m Word8 -> Stream m Char

#

Decode a UTF-8 encoded bytestream to a stream of Unicode characters. The function throws an error if an invalid codepoint is encountered.

valuedecodeUtf8_ :: Monad m => Stream m Word8 -> Stream m Char

#

Decode a UTF-8 encoded bytestream to a stream of Unicode characters. Any invalid codepoint encountered is dropped.

valuedecodeUtf16le' :: Stream m Word16 -> Stream m Char

#

Decode a UTF-16 little endian encoded bytestream to a stream of Unicode characters. The function throws an error if an invalid codepoint is encountered.

Unimplemented

Resumable UTF-8 Decoding

datadata DecodeError

#

Constructors

DecodeError !DecodeState !CodePoint

Instances1Show

Show DecodeErrorDefined in streamly-core-0.2.2 · Streamly.Internal.Unicode.Stream

typetype DecodeState = Word8

#

typetype CodePoint = Int

#

valuedecodeUtf8Either

:: Monad m
=> Stream m Word8
-> Stream m (Either DecodeError Char)

#

Pre-release

valueresumeDecodeUtf8Either

#

Pre-release

UTF-8 Array Stream Decoding

valuedecodeUtf8Chunks :: MonadIO m => Stream m (Array Word8) -> Stream m Char

#

Like decodeUtf8 but for a chunked stream. It may be slightly faster than flattening the stream and then decoding with decodeUtf8.

valuedecodeUtf8Chunks' :: MonadIO m => Stream m (Array Word8) -> Stream m Char

#

Like 'decodeUtf8'' but for a chunked stream. It may be slightly faster than flattening the stream and then decoding with 'decodeUtf8''.

valuedecodeUtf8Chunks_ :: MonadIO m => Stream m (Array Word8) -> Stream m Char

#

Like decodeUtf8_ but for a chunked stream. It may be slightly faster than flattening the stream and then decoding with decodeUtf8_.

Elimination (Encoding)

0 declarations

Latin1 Encoding

valueencodeLatin1 :: Monad m => Stream m Char -> Stream m Word8

#

Like encodeLatin1' but silently maps input codepoints beyond 255 to arbitrary Latin1 chars in 0-255 range. No error or exception is thrown when such mapping occurs.

valueencodeLatin1' :: Monad m => Stream m Char -> Stream m Word8

#

Encode a stream of Unicode characters to bytes by mapping each character to a byte in 0-255 range. Throws an error if the input stream contains characters beyond 255.

valueencodeLatin1_ :: Monad m => Stream m Char -> Stream m Word8

#

Like encodeLatin1 but drops the input characters beyond 255.

UTF-8 Encoding

valuereadCharUtf8' :: Monad m => Unfold m Char Word8

#

valuereadCharUtf8 :: Monad m => Unfold m Char Word8

#

valuereadCharUtf8_ :: Monad m => Unfold m Char Word8

#

valueencodeUtf8 :: Monad m => Stream m Char -> Stream m Word8

#

Encode a stream of Unicode characters to a UTF-8 encoded bytestream. Any Invalid characters (U+D800-U+D8FF) in the input stream are replaced by the Unicode replacement character U+FFFD.

valueencodeUtf8' :: Monad m => Stream m Char -> Stream m Word8

#

Encode a stream of Unicode characters to a UTF-8 encoded bytestream. When any invalid character (U+D800-U+D8FF) is encountered in the input stream the function errors out.

valueencodeUtf8_ :: Monad m => Stream m Char -> Stream m Word8

#

Encode a stream of Unicode characters to a UTF-8 encoded bytestream. Any Invalid characters (U+D800-U+D8FF) in the input stream are dropped.

valueencodeStrings

:: MonadIO m
=> Stream m Char -> Stream m Word8
-> Stream m String
-> Stream m (Array Word8)

#

Encode a stream of String using the supplied encoding scheme. Each string is encoded as an Array Word8.

valueencodeUtf16le' :: Stream m Char -> Stream m Word16

#

Encode a stream of Unicode characters to a UTF-16 little endian encoded bytestream.

Unimplemented

Transformation

5 declarations

valuestripHead :: Monad m => Stream m Char -> Stream m Char

#

Remove leading whitespace from a string.

stripHead = Stream.dropWhile isSpace

Pre-release

valuelines :: Monad m => Fold m Char b -> Stream m Char -> Stream m b

#

Fold each line of the stream using the supplied Fold and stream the result.

Example1 expression

>>> Stream.fold Fold.toList $ Unicode.lines Fold.toList (Stream.fromList "lines\nthis\nstring\n\n\n")["lines","this","string","",""]

lines = Stream.splitOnSuffix (== '\n')

Pre-release

valuewords :: Monad m => Fold m Char b -> Stream m Char -> Stream m b

#

Fold each word of the stream using the supplied Fold and stream the result.

Example1 expression

>>> Stream.fold Fold.toList $ Unicode.words Fold.toList (Stream.fromList "fold these     words")["fold","these","words"]

words = Stream.wordsBy isSpace

Pre-release

valueunlines :: MonadIO m => Unfold m a Char -> Stream m a -> Stream m Char

#

Unfold a stream to character streams using the supplied Unfold and concat the results suffixing a newline character \n to each stream.

unlines = Stream.interposeSuffix 'n'
unlines = Stream.intercalateSuffix Unfold.fromList "n"

Pre-release

valueunwords :: MonadIO m => Unfold m a Char -> Stream m a -> Stream m Char

#

Unfold the elements of a stream to character streams using the supplied Unfold and concat the results with a whitespace character infixed between the streams.

unwords = Stream.interpose ' '
unwords = Stream.intercalate Unfold.fromList " "

Pre-release

StreamD UTF8 Encoding / Decoding transformations.

8 declarations

valuedecodeUtf8D :: Monad m => Stream m Word8 -> Stream m Char

#

valuedecodeUtf8D' :: Monad m => Stream m Word8 -> Stream m Char

#

valuedecodeUtf8D_ :: Monad m => Stream m Word8 -> Stream m Char

#

valueencodeUtf8D :: Monad m => Stream m Char -> Stream m Word8

#

See section "3.9 Unicode Encoding Forms" in https://www.unicode.org/versions/Unicode13.0.0/UnicodeStandard-13.0.pdf

valueencodeUtf8D' :: Monad m => Stream m Char -> Stream m Word8

#

valueencodeUtf8D_ :: Monad m => Stream m Char -> Stream m Word8

#

valuedecodeUtf8EitherD

:: Monad m
=> Stream m Word8
-> Stream m (Either DecodeError Char)

#

valueresumeDecodeUtf8EitherD

#

Decoding String Literals

1 declaration

valuefromStr# :: MonadIO m => Addr# -> Stream m Char

#

Read UTF-8 encoded bytes as chars from an Addr# until a 0 byte is encountered, the 0 byte is not included in the stream.

Unsafe: The caller is responsible for safe addressing.

Note that this is completely safe when reading from Haskell string literals because they are guaranteed to be NULL terminated:

Example1 expression

>>> Stream.fold Fold.toList (Unicode.fromStr# "Haskell"#)"Haskell"

Deprecations

3 declarations

valuedecodeUtf8Lax :: Monad m => Stream m Word8 -> Stream m Char

#

Deprecated. Please use decodeUtf8 instead

Same as decodeUtf8

valueencodeLatin1Lax :: Monad m => Stream m Char -> Stream m Word8

#

Deprecated. Please use encodeLatin1 instead

Same as encodeLatin1

valueencodeUtf8Lax :: Monad m => Stream m Char -> Stream m Word8

#

Deprecated. Please use encodeUtf8 instead

Same as encodeUtf8