jamosdev
/
rhubarb-lip-sync
mirror of https://github.com/DanielSWolf/rhubarb-lip-sync.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #18 from DanielSWolf/feature/utf-8 

Full Unicode support by using UTF-8 throughout
This commit is contained in:
Daniel Wolf 2017-09-10 22:10:24 +02:00 committed by GitHub
parent 98fb5e7521 1fb2eb9024
commit 4de8f3d18e
73 changed files with 23161 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
CMakeLists.txt
View File

@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 3.2)
# Support legacy OS X versions
set(CMAKE_OSX_DEPLOYMENT_TARGET "10.7" CACHE STRING "Minimum OS X deployment version")
set(CMAKE_OSX_DEPLOYMENT_TARGET "10.8" CACHE STRING "Minimum OS X deployment version")
set(appName "Rhubarb Lip Sync")
set(appVersionMajor 1)
@ -49,6 +49,11 @@ elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /utf-8")
endif()
# Use UTF-8 throughout
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
add_compile_options("/utf-8")
endif()
if(${UNIX})
add_definitions(-DHAVE_UNISTD_H)
endif()
@ -214,6 +219,24 @@ target_include_directories(flite SYSTEM PUBLIC
target_compile_options(flite PRIVATE ${disableWarningsFlags})
set_target_properties(flite PROPERTIES FOLDER lib)
# ... UTF8-CPP
add_library(utfcpp
lib/header-only.c
lib/utfcpp-2.3.5/source/utf8.h
)
target_include_directories(utfcpp SYSTEM PUBLIC "lib/utfcpp-2.3.5/source")
target_compile_options(utfcpp PRIVATE ${disableWarningsFlags})
set_target_properties(utfcpp PROPERTIES FOLDER lib)
# ... utf8proc
add_library(utf8proc
lib/utf8proc-2a2f97e1/utf8proc.c
lib/utf8proc-2a2f97e1/utf8proc.h
)
target_include_directories(utf8proc SYSTEM PUBLIC "lib/utf8proc-2a2f97e1")
target_compile_options(utf8proc PRIVATE ${disableWarningsFlags})
set_target_properties(utf8proc PROPERTIES FOLDER lib)
# Define Rhubarb libraries
# ... rhubarb-animation
@ -401,6 +424,8 @@ target_include_directories(rhubarb-tools PUBLIC "src/tools")
target_link_libraries(rhubarb-tools
cppFormat
whereami
utfcpp
utf8proc
)
# Define Rhubarb executable

54
LICENSE.md
View File

@ -162,6 +162,60 @@ The [TCLAP](http://tclap.sourceforge.net/) library is released under the **MIT L
>
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
### `[utfcpp]` UTF8-CPP
The [UTF8-CPP](https://github.com/nemtrif/utfcpp) library is released under the **Boost Software License**.
> Copyright 2006 Nemanja Trifunovic
>
>Permission is hereby granted, free of charge, to any person or organization obtaining a copy of the software and accompanying documentation covered by this license (the "Software") to use, reproduce, display, distribute, execute, and transmit the Software, and to prepare derivative works of the Software, and to permit third-parties to whom the Software is furnished to do so, all subject to the following:
>
> The copyright notices in the Software and this entire statement, including the above license grant, this restriction and the following disclaimer, must be included in all copies of the Software, in whole or in part, and all derivative works of the Software, unless such copies or derivative works are solely in the form of machine-executable object code generated by a source language processor.
>
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
### `[utf8proc]` utf8proc
The [utf8proc](https://github.com/JuliaLang/utf8proc) library is released under the **MIT License (MIT)**, while some of its data is released under the **UNICODE License**.
> #### utf8proc license
>
> **utf8proc** is a software package originally developed by Jan Behrens and the rest of the Public Software Group, who deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers. Like the original utf8proc, whose copyright and license statements are reproduced below, all new work on the utf8proc library is licensed under the [MIT "expat" license](http://opensource.org/licenses/MIT):
>
> *Copyright © 2014-2015 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.*
>
> Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
>
> The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
>
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
>
> #### Original utf8proc license
>
> *Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany*
>
> Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
>
> The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
>
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
>
> #### Unicode data license
>
> This software distribution contains derived data from a modified version of the Unicode data files. The following license applies to that data:
>
> **COPYRIGHT AND PERMISSION NOTICE**
>
> *Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed under the Terms of Use in http://www.unicode.org/copyright.html.*
>
> Permission is hereby granted, free of charge, to any person obtaining a copy of the Unicode data files and any associated documentation (the "Data Files") or Unicode software and any associated documentation (the "Software") to deal in the Data Files or Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, and/or sell copies of the Data Files or Software, and to permit persons to whom the Data Files or Software are furnished to do so, provided that (a) the above copyright notice(s) and this permission notice appear with all copies of the Data Files or Software, (b) both the above copyright notice(s) and this permission notice appear in associated documentation, and (c) there is clear notice in each modified Data File or in the Software as well as in the documentation associated with the Data File(s) or Software that the data or software has been modified.
>
> THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA FILES OR SOFTWARE.
>
> Except as contained in this notice, the name of a copyright holder shall not be used in advertising or otherwise to promote the sale, use or other dealings in these Data Files or Software without prior written authorization of the copyright holder.
>
> Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be registered in some jurisdictions. All other trademarks and registered trademarks mentioned herein are the property of their respective owners.
### `[webrtc]` WebRTC
The [WebRTC](https://chromium.googlesource.com/external/webrtc) library is released under the **3-clause BSD License**.

7
lib/header-only.c Normal file
View File

@ -0,0 +1,7 @@
/*
This library is header-only.
Due to limitations in CMake, I added this dummy code file.
For details, see http://cmake.3232098.n2.nabble.com/Cannot-set-FOLDER-property-to-an-interface-header-only-target-td7592375.html
*/

28
lib/utf8proc-2a2f97e1/.gitignore vendored Normal file
View File

@ -0,0 +1,28 @@
*.tar.gz
*.exe
*.dll
*.do
*.o
*.so*
*.a
*.dll
*.dylib
*.dSYM
*.out
*.new
data/*.txt
data/*.ttf
data/*.sfd
/docs/
bench/bench
bench/icu
bench/unistring
test/normtest
test/graphemetest
test/printproperty
test/charwidth
test/valid
test/iterate
test/case
test/custom
/tmp/

22
lib/utf8proc-2a2f97e1/.travis.yml Normal file
View File

@ -0,0 +1,22 @@
language: c
compiler:
- gcc
- clang
notifications:
email: false
before_install:
- sudo add-apt-repository ppa:staticfloat/julia-deps -y
- sudo add-apt-repository ppa:staticfloat/juliareleases -y
- sudo apt-get update -qq -y
- sudo apt-get install libpcre3-dev julia fontforge -y
script:
- make manifest && diff MANIFEST.new MANIFEST
- make check
- make data && diff data/utf8proc_data.c.new utf8proc_data.c
- make clean && git status --ignored --porcelain && test -z "$(git status --ignored --porcelain)"
- (mkdir build_static && cd build_static && cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON && make)
- (mkdir build_shared && cd build_shared && cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DBUILD_SHARED_LIBS=ON && make)
env:
# use JuliaLang caching (https://github.com/staticfloat/cache.julialang.org)
# so that Travis builds do not depend on anyone's flaky servers but our own
- URLCACHE=https://cache.julialang.org/ CFLAGS="-O2 -Werror -Wmissing-prototypes"

33
lib/utf8proc-2a2f97e1/CMakeLists.txt Normal file
View File

@ -0,0 +1,33 @@
cmake_minimum_required (VERSION 2.8)
include (utils.cmake)
disallow_intree_builds()
project (utf8proc C)
# This is the ABI version number, which may differ from the
# API version number (defined in utf8proc.h).
# Be sure to also update these in Makefile and MANIFEST!
set(SO_MAJOR 2)
set(SO_MINOR 1)
set(SO_PATCH 0)
add_definitions (
-DUTF8PROC_EXPORTS
)
if (NOT MSVC)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -std=c99 -pedantic -Wall")
endif ()
add_library (utf8proc
utf8proc.c
utf8proc.h
)
set_target_properties (utf8proc PROPERTIES
POSITION_INDEPENDENT_CODE ON
VERSION "${SO_MAJOR}.${SO_MINOR}.${SO_PATCH}"
SOVERSION ${SO_MAJOR}
)

1528
lib/utf8proc-2a2f97e1/Doxyfile Normal file
View File

File diff suppressed because it is too large Load Diff

93
lib/utf8proc-2a2f97e1/LICENSE.md Normal file
View File

@ -0,0 +1,93 @@
## utf8proc license ##
**utf8proc** is a software package originally developed
by Jan Behrens and the rest of the Public Software Group, who
deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers. Like the original utf8proc,
whose copyright and license statements are reproduced below, all new
work on the utf8proc library is licensed under the [MIT "expat"
license](http://opensource.org/licenses/MIT):
*Copyright © 2014-2015 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.*
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
## Original utf8proc license ##
*Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany*
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
## Unicode data license ##
This software distribution contains derived data from a modified version of
the Unicode data files. The following license applies to that data:
**COPYRIGHT AND PERMISSION NOTICE**
*Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
under the Terms of Use in http://www.unicode.org/copyright.html.*
Permission is hereby granted, free of charge, to any person obtaining a
copy of the Unicode data files and any associated documentation (the "Data
Files") or Unicode software and any associated documentation (the
"Software") to deal in the Data Files or Software without restriction,
including without limitation the rights to use, copy, modify, merge,
publish, distribute, and/or sell copies of the Data Files or Software, and
to permit persons to whom the Data Files or Software are furnished to do
so, provided that (a) the above copyright notice(s) and this permission
notice appear with all copies of the Data Files or Software, (b) both the
above copyright notice(s) and this permission notice appear in associated
documentation, and (c) there is clear notice in each modified Data File or
in the Software as well as in the documentation associated with the Data
File(s) or Software that the data or software has been modified.
THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR
CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THE DATA FILES OR SOFTWARE.
Except as contained in this notice, the name of a copyright holder shall
not be used in advertising or otherwise to promote the sale, use or other
dealings in these Data Files or Software without prior written
authorization of the copyright holder.
Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be
registered in some jurisdictions. All other trademarks and registered
trademarks mentioned herein are the property of their respective owners.

7
lib/utf8proc-2a2f97e1/MANIFEST Normal file
View File

@ -0,0 +1,7 @@
include/
include/utf8proc.h
lib/
lib/libutf8proc.a
lib/libutf8proc.so -> libutf8proc.so.2.1.0
lib/libutf8proc.so.2 -> libutf8proc.so.2.1.0
lib/libutf8proc.so.2.1.0

149
lib/utf8proc-2a2f97e1/Makefile Normal file
View File

@ -0,0 +1,149 @@
# libutf8proc Makefile
# programs
AR?=ar
CC?=gcc
INSTALL=install
FIND=find
# compiler settings
CFLAGS ?= -O2
PICFLAG = -fPIC
C99FLAG = -std=c99
WCFLAGS = -Wall -pedantic
UCFLAGS = $(CFLAGS) $(PICFLAG) $(C99FLAG) $(WCFLAGS) -DUTF8PROC_EXPORTS
# shared-library version MAJOR.MINOR.PATCH ... this may be *different*
# from the utf8proc version number because it indicates ABI compatibility,
# not API compatibility: MAJOR should be incremented whenever *binary*
# compatibility is broken, even if the API is backward-compatible.
# The API version number is defined in utf8proc.h.
# Be sure to also update these ABI versions in MANIFEST and CMakeLists.txt!
MAJOR=2
MINOR=1
PATCH=0
OS := $(shell uname)
ifeq ($(OS),Darwin) # MacOS X
SHLIB_EXT = dylib
SHLIB_VERS_EXT = $(MAJOR).dylib
else # GNU/Linux, at least (Windows should probably use cmake)
SHLIB_EXT = so
SHLIB_VERS_EXT = so.$(MAJOR).$(MINOR).$(PATCH)
endif
# installation directories (for 'make install')
prefix=/usr/local
libdir=$(prefix)/lib
includedir=$(prefix)/include
# meta targets
.PHONY: all clean data update manifest install
all: libutf8proc.a libutf8proc.$(SHLIB_EXT)
clean:
rm -f utf8proc.o libutf8proc.a libutf8proc.$(SHLIB_VERS_EXT) libutf8proc.$(SHLIB_EXT)
ifneq ($(OS),Darwin)
rm -f libutf8proc.so.$(MAJOR)
endif
rm -f test/tests.o test/normtest test/graphemetest test/printproperty test/charwidth test/valid test/iterate test/case test/custom
rm -rf MANIFEST.new tmp
$(MAKE) -C bench clean
$(MAKE) -C data clean
data: data/utf8proc_data.c.new
update: data/utf8proc_data.c.new
cp -f data/utf8proc_data.c.new utf8proc_data.c
manifest: MANIFEST.new
# real targets
data/utf8proc_data.c.new: libutf8proc.$(SHLIB_EXT) data/data_generator.rb data/charwidths.jl
$(MAKE) -C data utf8proc_data.c.new
utf8proc.o: utf8proc.h utf8proc.c utf8proc_data.c
$(CC) $(UCFLAGS) -c -o utf8proc.o utf8proc.c
libutf8proc.a: utf8proc.o
rm -f libutf8proc.a
$(AR) rs libutf8proc.a utf8proc.o
libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH): utf8proc.o
$(CC) $(LDFLAGS) -shared -o $@ -Wl,-soname -Wl,libutf8proc.so.$(MAJOR) utf8proc.o
chmod a-x $@
libutf8proc.so: libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH)
ln -f -s libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH) $@
ln -f -s libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH) $@.$(MAJOR)
libutf8proc.$(MAJOR).dylib: utf8proc.o
$(CC) -dynamiclib -o $@ $^ -install_name $(libdir)/$@ -Wl,-compatibility_version -Wl,$(MAJOR) -Wl,-current_version -Wl,$(MAJOR).$(MINOR).$(PATCH)
libutf8proc.dylib: libutf8proc.$(MAJOR).dylib
ln -f -s libutf8proc.$(MAJOR).dylib $@
install: libutf8proc.a libutf8proc.$(SHLIB_EXT) libutf8proc.$(SHLIB_VERS_EXT)
mkdir -m 755 -p $(DESTDIR)$(includedir)
$(INSTALL) -m 644 utf8proc.h $(DESTDIR)$(includedir)
mkdir -m 755 -p $(DESTDIR)$(libdir)
$(INSTALL) -m 644 libutf8proc.a $(DESTDIR)$(libdir)
$(INSTALL) -m 755 libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)
ln -f -s libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)/libutf8proc.$(SHLIB_EXT)
ifneq ($(OS),Darwin)
ln -f -s libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)/libutf8proc.so.$(MAJOR)
endif
MANIFEST.new:
rm -rf tmp
$(MAKE) install prefix=/usr DESTDIR=$(PWD)/tmp
$(FIND) tmp/usr -mindepth 1 -type l -printf "%P -> %l\n" -or -type f -printf "%P\n" -or -type d -printf "%P/\n" | LC_ALL=C sort > $@
rm -rf tmp
# Test programs
data/NormalizationTest.txt:
$(MAKE) -C data NormalizationTest.txt
data/GraphemeBreakTest.txt:
$(MAKE) -C data GraphemeBreakTest.txt
test/tests.o: test/tests.c test/tests.h utf8proc.h
$(CC) $(UCFLAGS) -c -o test/tests.o test/tests.c
test/normtest: test/normtest.c test/tests.o utf8proc.o utf8proc.h test/tests.h
$(CC) $(UCFLAGS) test/normtest.c test/tests.o utf8proc.o -o $@
test/graphemetest: test/graphemetest.c test/tests.o utf8proc.o utf8proc.h test/tests.h
$(CC) $(UCFLAGS) test/graphemetest.c test/tests.o utf8proc.o -o $@
test/printproperty: test/printproperty.c test/tests.o utf8proc.o utf8proc.h test/tests.h
$(CC) $(UCFLAGS) test/printproperty.c test/tests.o utf8proc.o -o $@
test/charwidth: test/charwidth.c test/tests.o utf8proc.o utf8proc.h test/tests.h
$(CC) $(UCFLAGS) test/charwidth.c test/tests.o utf8proc.o -o $@
test/valid: test/valid.c test/tests.o utf8proc.o utf8proc.h test/tests.h
$(CC) $(UCFLAGS) test/valid.c test/tests.o utf8proc.o -o $@
test/iterate: test/iterate.c test/tests.o utf8proc.o utf8proc.h test/tests.h
$(CC) $(UCFLAGS) test/iterate.c test/tests.o utf8proc.o -o $@
test/case: test/case.c test/tests.o utf8proc.o utf8proc.h test/tests.h
$(CC) $(UCFLAGS) test/case.c test/tests.o utf8proc.o -o $@
test/custom: test/custom.c test/tests.o utf8proc.o utf8proc.h test/tests.h
$(CC) $(UCFLAGS) test/custom.c test/tests.o utf8proc.o -o $@
check: test/normtest data/NormalizationTest.txt test/graphemetest data/GraphemeBreakTest.txt test/printproperty test/case test/custom test/charwidth test/valid test/iterate bench/bench.c bench/util.c bench/util.h utf8proc.o
$(MAKE) -C bench
test/normtest data/NormalizationTest.txt
test/graphemetest data/GraphemeBreakTest.txt
test/charwidth
test/valid
test/iterate
test/case
test/custom

303
lib/utf8proc-2a2f97e1/NEWS.md Normal file
View File

@ -0,0 +1,303 @@
# utf8proc release history #
## Version 2.1 ##
2016-12-26:
- New functions `utf8proc_map_custom` and `utf8proc_decompose_custom`
to allow user-supplied transformations of codepoints, in conjunction
with other transformations ([#89]).
- New function `utf8proc_normalize_utf32` to apply normalizations
directly to UTF-32 data (not just UTF-8) ([#88]).
- Fixed stack overflow that could occur due to incorrect definition
of `UINT16_MAX` with some compilers ([#84]).
- Fixed conflict with `stdbool.h` in Visual Studio ([#90]).
- Updated font metrics to use Unifont 9.0.04.
## Version 2.0.2 ##
2016-07-27:
- Move `-Wmissing-prototypes` warning flag from `Makefile` to `.travis.yml`
since MSVC does not understand this flag and it is occasionally useful to
build using MSVC through the `Makefile` ([#79]).
- Use a different variable name for a nested loop in `bench/bench.c`, and
declare it in a C89 way rather than inside the `for` to avoid "error:
'for' loop initial declarations are only allowed in C99 mode" ([#80]).
## Version 2.0.1 ##
2016-07-13:
- Bug fix in `utf8proc_grapheme_break_stateful` ([#77]).
- Tests now use versioned Unicode files, so they will no longer
break when a new version of Unicode is released ([#78]).
## Version 2.0 ##
2016-07-13:
- Updated for Unicode 9.0 ([#70]).
- New `utf8proc_grapheme_break_stateful` to handle the complicated
grapheme-breaking rules in Unicode 9. The old `utf8proc_grapheme_break`
is still provided, but may incorrectly identify grapheme breaks
in some Unicode-9 sequences.
- Smaller Unicode tables ([#62], [#68]). This required changes
in the `utf8proc_property_t` structure, which breaks backward
compatibility if you access this `struct` directly. The
functions in the API remain backward-compatible, however.
- Buffer overrun fix ([#66]).
## Version 1.3.1 ##
2015-11-02:
- Do not export symbol for internal function `unsafe_encode_char()` ([#55]).
- Install relative symbolic links for shared libraries ([#58]).
- Enable and fix compiler warnings ([#55], [#58]).
- Add missing files to `make clean` ([#58]).
## Version 1.3 ##
2015-07-06:
- Updated for Unicode 8.0 ([#45]).
- New `utf8proc_tolower` and `utf8proc_toupper` functions, portable
replacements for `towlower` and `towupper` in the C library ([#40]).
- Don't treat Unicode "non-characters" as invalid, and improved
validity checking in general ([#35]).
- Prefix all typedefs with `utf8proc_`, e.g. `utf8proc_int32_t`,
to avoid collisions with other libraries ([#32]).
- Rename `DLLEXPORT` to `UTF8PROC_DLLEXPORT` to prevent collisions.
- Fix build breakage in the benchmark routines.
- More fine-grained Makefile variables (`PICFLAG` etcetera), so that
compilation flags can be selectively overridden, and in particular
so that `CFLAGS` can be changed without accidentally eliminating
necessary flags like `-fPIC` and `-std=c99` ([#43]).
- Updated character-width tables based on Unifont 8.0.01 ([#51]) and
the Unicode 8 character categories ([#47]).
## Version 1.2 ##
2015-03-28:
- Updated for Unicode 7.0 ([#6]).
- New function `utf8proc_grapheme_break(c1,c2)` that returns whether
there is a grapheme break between `c1` and `c2` ([#20]).
- New function `utf8proc_charwidth(c)` that returns the number of
column-positions that should be required for `c`; essentially a
portable replacment for `wcwidth(c)` ([#27]).
- New function `utf8proc_category(c)` that returns the Unicode
category of `c` (as one of the constants `UTF8PROC_CATEGORY_xx`).
Also, a function `utf8proc_category_string(c)` that returns the Unicode
category of `c` as a two-character string.
- `cmake` script `CMakeLists.txt`, in addition to `Makefile`, for
easier compilation on Windows ([#28]).
- Various `Makefile` improvements: a `make check` target to perform
tests ([#13]), `make install`, a rule to automate updating the Unicode
tables, etcetera.
- The shared library is now versioned (e.g. has a soname on GNU/Linux) ([#24]).
- C++/MSVC compatibility ([#17]).
- Most `#defined` constants are now `enums` ([#29]).
- New preprocessor constants `UTF8PROC_VERSION_MAJOR`,
`UTF8PROC_VERSION_MINOR`, and `UTF8PROC_VERSION_PATCH` for compile-time
detection of the API version.
- Doxygen-formatted documentation ([#29]).
- The Ruby and PostgreSQL plugins have been removed due to lack of testing ([#22]).
## Version 1.1.6 ##
2013-11-27:
- PostgreSQL 9.2 and 9.3 compatibility (lowercase `c` language name)
## Version 1.1.5 ##
2009-08-20:
- Use `RSTRING_PTR()` and `RSTRING_LEN()` instead of `RSTRING()->ptr` and
`RSTRING()->len` for ruby1.9 compatibility (and `#define` them, if not
existent)
2009-10-02:
- Patches for compatibility with Microsoft Visual Studio
2009-10-08:
- Fixes to make utf8proc usable in C++ programs
2009-10-16:
## Version 1.1.4 ##
2009-06-14:
- replaced C++ style comments for compatibility reasons
- added typecasts to suppress compiler warnings
- removed redundant source files for ruby-gemfile generation
2009-08-19:
- Changed copyright notice for Public Software Group e. V.
- Minor changes in the `README` file
## Version 1.1.3 ##
2008-10-04:
- Added a function `utf8proc_version` returning a string containing the version
number of the library.
- Included a target `libutf8proc.dylib` for MacOSX.
2009-05-01:
- PostgreSQL 8.3 compatibility (use of `SET_VARSIZE` macro)
## Version 1.1.2 ##
2007-07-25:
- Fixed a serious bug in the data file generator, which caused characters
being treated incorrectly, when stripping default ignorable characters or
calculating grapheme cluster boundaries.
## Version 1.1.1 ##
2007-06-25:
- Added a new PostgreSQL function `unistrip`, which behaves like `unifold`,
but also removes all character marks (e.g. accents).
2007-07-22:
- Changed license from BSD to MIT style.
- Added a new function `utf8proc_codepoint_valid` to the C library.
- Changed compiler flags in `Makefile` from `-g -O0` to `-O2`
- The ruby script, which was used to build the `utf8proc_data.c` file, is now
included in the distribution.
## Version 1.0.3 ##
2007-03-16:
- Fixed a bug in the ruby library, which caused an error, when splitting an
empty string at grapheme cluster boundaries (method `String#utf8chars`).
## Version 1.0.2 ##
2006-09-21:
- included a check in `Integer#utf8`, which raises an exception, if the given
code-point is invalid because of being too high (this was missing yet)
2006-12-26:
- added support for PostgreSQL version 8.2
## Version 1.0.1 ##
2006-09-20:
- included a gem file for the ruby version of the library
Release of version 1.0.1
## Version 1.0 ##
2006-09-17:
- added the `LUMP` option, which lumps certain characters together (see `lump.md`) (also used for the PostgreSQL `unifold` function)
- added the `STRIPMARK` option, which strips marking characters (or marks of composed characters)
- deprecated ruby method `String#char_ary` in favour of `String#utf8chars`
## Version 0.3 ##
2006-07-18:
- changed normalization from NFC to NFKC for postgresql unifold function
2006-08-04:
- added support to mark the beginning of a grapheme cluster with 0xFF (option: `CHARBOUND`)
- added the ruby method `String#chars`, which is returning an array of UTF-8 encoded grapheme clusters
- added `NLF2LF` transformation in postgresql `unifold` function
- added the `DECOMPOSE` option, if you neither use `COMPOSE` or `DECOMPOSE`, no normalization will be performed (different from previous versions)
- using integer constants rather than C-strings for character properties
- fixed (hopefully) a problem with the ruby library on Mac OS X, which occurred when compiler optimization was switched on
## Version 0.2 ##
2006-06-05:
- changed behaviour of PostgreSQL function to return NULL in case of invalid input, rather than raising an exceptional condition
- improved efficiency of PostgreSQL function (no transformation to C string is done)
2006-06-20:
- added -fpic compiler flag in Makefile
- fixed bug in the C code for the ruby library (usage of non-existent function)
## Version 0.1 ##
2006-06-02: initial release of version 0.1
[#6]: https://github.com/JuliaLang/utf8proc/issues/6
[#13]: https://github.com/JuliaLang/utf8proc/issues/13
[#17]: https://github.com/JuliaLang/utf8proc/issues/17
[#20]: https://github.com/JuliaLang/utf8proc/issues/20
[#22]: https://github.com/JuliaLang/utf8proc/issues/22
[#24]: https://github.com/JuliaLang/utf8proc/issues/24
[#27]: https://github.com/JuliaLang/utf8proc/issues/27
[#28]: https://github.com/JuliaLang/utf8proc/issues/28
[#29]: https://github.com/JuliaLang/utf8proc/issues/29
[#32]: https://github.com/JuliaLang/utf8proc/issues/32
[#35]: https://github.com/JuliaLang/utf8proc/issues/35
[#40]: https://github.com/JuliaLang/utf8proc/issues/40
[#43]: https://github.com/JuliaLang/utf8proc/issues/43
[#45]: https://github.com/JuliaLang/utf8proc/issues/45
[#47]: https://github.com/JuliaLang/utf8proc/issues/47
[#51]: https://github.com/JuliaLang/utf8proc/issues/51
[#55]: https://github.com/JuliaLang/utf8proc/issues/55
[#58]: https://github.com/JuliaLang/utf8proc/issues/58
[#62]: https://github.com/JuliaLang/utf8proc/issues/62
[#66]: https://github.com/JuliaLang/utf8proc/issues/66
[#68]: https://github.com/JuliaLang/utf8proc/issues/68
[#70]: https://github.com/JuliaLang/utf8proc/issues/70
[#77]: https://github.com/JuliaLang/utf8proc/issues/77
[#78]: https://github.com/JuliaLang/utf8proc/issues/78
[#79]: https://github.com/JuliaLang/utf8proc/issues/79
[#80]: https://github.com/JuliaLang/utf8proc/issues/80
[#84]: https://github.com/JuliaLang/utf8proc/pull/84
[#88]: https://github.com/JuliaLang/utf8proc/pull/88
[#89]: https://github.com/JuliaLang/utf8proc/pull/89
[#90]: https://github.com/JuliaLang/utf8proc/issues/90

69
lib/utf8proc-2a2f97e1/README.md Normal file
View File

@ -0,0 +1,69 @@
# utf8proc
[![Travis CI Status](https://travis-ci.org/JuliaLang/utf8proc.png)](https://travis-ci.org/JuliaLang/utf8proc)
[![AppVeyor Status](https://ci.appveyor.com/api/projects/status/aou20lfkyhj8xbwq/branch/master?svg=true)](https://ci.appveyor.com/project/tkelman/utf8proc/branch/master)
[utf8proc](http://julialang.org/utf8proc/) is a small, clean C
library that provides Unicode normalization, case-folding, and other
operations for data in the [UTF-8
encoding](http://en.wikipedia.org/wiki/UTF-8). It was [initially
developed](http://www.public-software-group.org/utf8proc) by Jan
Behrens and the rest of the [Public Software
Group](http://www.public-software-group.org/), who deserve *nearly all
of the credit* for this package. With the blessing of the Public
Software Group, the [Julia developers](http://julialang.org/) have
taken over development of utf8proc, since the original developers have
moved to other projects.
(utf8proc is used for basic Unicode
support in the [Julia language](http://julialang.org/), and the Julia
developers became involved because they wanted to add Unicode 7 support and other features.)
(The original utf8proc package also includes Ruby and PostgreSQL plug-ins.
We removed those from utf8proc in order to focus exclusively on the C
library for the time being, but plan to add them back in or release them as separate packages.)
The utf8proc package is licensed under the
free/open-source [MIT "expat"
license](http://opensource.org/licenses/MIT) (plus certain Unicode
data governed by the similarly permissive [Unicode data
license](http://www.unicode.org/copyright.html#Exhibit1)); please see
the included `LICENSE.md` file for more detailed information.
## Quick Start
For compilation of the C library run `make`.
## General Information
The C library is found in this directory after successful compilation
and is named `libutf8proc.a` (for the static library) and
`libutf8proc.so` (for the dynamic library).
The Unicode version supported is 9.0.0.
For Unicode normalizations, the following options are used:
* Normalization Form C: `STABLE`, `COMPOSE`
* Normalization Form D: `STABLE`, `DECOMPOSE`
* Normalization Form KC: `STABLE`, `COMPOSE`, `COMPAT`
* Normalization Form KD: `STABLE`, `DECOMPOSE`, `COMPAT`
## C Library
The documentation for the C library is found in the `utf8proc.h` header file.
`utf8proc_map` is function you will most likely be using for mapping UTF-8
strings, unless you want to allocate memory yourself.
## To Do
See the Github [issues list](https://github.com/JuliaLang/utf8proc/issues).
## Contact
Bug reports, feature requests, and other queries can be filed at
the [utf8proc issues page on Github](https://github.com/JuliaLang/utf8proc/issues).
## See also
An independent Lua translation of this library, [lua-mojibake](https://github.com/differentprogramming/lua-mojibake), is also available.

42
lib/utf8proc-2a2f97e1/appveyor.yml Normal file
View File

@ -0,0 +1,42 @@
branches:
only:
- master
- /release-.*/
notifications:
- provider: Email
on_build_success: false
on_build_failure: false
on_build_status_changed: false
build_script:
- ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
throw "There are newer queued builds for this pull request, failing early." }
- mkdir msvc_static
- cd msvc_static
- cmake ..
- cmake --build .
- mkdir ..\msvc_shared
- cd ..\msvc_shared
- cmake .. -DBUILD_SHARED_LIBS=ON
- cmake --build .
- set PATH=C:\MinGW\bin;%PATH%
- C:\MinGW\msys\1.0\bin\sh --login -c "
cd /c/projects/utf8proc &&
mkdir mingw_static &&
cd mingw_static &&
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -G'MSYS Makefiles' &&
make &&
mkdir ../mingw_shared &&
cd ../mingw_shared &&
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DBUILD_SHARED_LIBS=ON -G'MSYS Makefiles' &&
make
"
on_finish:
# Uncomment the following line for interactive debugging, which
# will print login data for a temporary remote session after the
# build. This requires an RDP version 6 client, e.g., FreeRDP.
#- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))

39
lib/utf8proc-2a2f97e1/bench/Makefile Normal file
View File

@ -0,0 +1,39 @@
CURL=curl
CC = cc
CFLAGS = -O2 -std=c99 -pedantic -Wall
all: bench
LIBUTF8PROC = ../utf8proc.o
bench: bench.o util.o $(LIBUTF8PROC)
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ bench.o util.o $(LIBUTF8PROC)
DATAURL = https://raw.githubusercontent.com/duerst/eprun/master/benchmark
DATAFILES = Deutsch_.txt Japanese_.txt Korean_.txt Vietnamese_.txt
$(DATAFILES):
$(CURL) -O $(DATAURL)/$@
bench.out: $(DATAFILES) bench
./bench -nfkc $(DATAFILES) > $@
# you may need make CPPFLAGS=... LDFLAGS=... to help it find ICU
icu: icu.o util.o
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ icu.o util.o -licuuc
icu.out: $(DATAFILES) icu
./icu $(DATAFILES) > $@
unistring: unistring.o util.o
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ unistring.o util.o -lunistring
unistring.out: $(DATAFILES) unistring
./unistring $(DATAFILES) > $@
.c.o:
$(CC) $(CPPFLAGS) -I.. $(CFLAGS) -c -o $@ $<
clean:
rm -rf *.o *.txt bench *.out icu unistring

56
lib/utf8proc-2a2f97e1/bench/bench.c Normal file
View File

@ -0,0 +1,56 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "utf8proc.h"
#include "util.h"
int main(int argc, char **argv)
{
int i, j;
int options = 0;
for (i = 1; i < argc; ++i) {
if (!strcmp(argv[i], "-nfkc")) {
options |= UTF8PROC_STABLE|UTF8PROC_COMPOSE|UTF8PROC_COMPAT;
continue;
}
if (!strcmp(argv[i], "-nfkd")) {
options |= UTF8PROC_STABLE|UTF8PROC_DECOMPOSE|UTF8PROC_COMPAT;
continue;
}
if (!strcmp(argv[i], "-nfc")) {
options |= UTF8PROC_STABLE|UTF8PROC_COMPOSE;
continue;
}
if (!strcmp(argv[i], "-nfd")) {
options |= UTF8PROC_STABLE|UTF8PROC_DECOMPOSE;
continue;
}
if (!strcmp(argv[i], "-casefold")) {
options |= UTF8PROC_CASEFOLD;
continue;
}
if (argv[i][0] == '-') {
fprintf(stderr, "unrecognized option: %s\n", argv[i]);
return EXIT_FAILURE;
}
size_t len;
uint8_t *src = readfile(argv[i], &len);
if (!src) {
fprintf(stderr, "error reading %s\n", argv[i]);
return EXIT_FAILURE;
}
uint8_t *dest;
mytime start = gettime();
for (j = 0; j < 100; ++j) {
utf8proc_map(src, len, &dest, options);
free(dest);
}
printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
free(src);
}
return EXIT_SUCCESS;
}

61
lib/utf8proc-2a2f97e1/bench/icu.c Normal file
View File

@ -0,0 +1,61 @@
#include <stdio.h>
#include <stdlib.h>
/* ICU4C */
#include <unicode/utypes.h>
#include <unicode/ustring.h>
#include <unicode/ucnv.h>
#include <unicode/unorm2.h>
#include "util.h"
int main(int argc, char **argv)
{
int i;
UErrorCode err;
UConverter *uc = ucnv_open("UTF8", &err);
if (U_FAILURE(err)) return EXIT_FAILURE;
const UNormalizer2 *NFKC = unorm2_getNFKCInstance(&err);
if (U_FAILURE(err)) return EXIT_FAILURE;
for (i = 1; i < argc; ++i) {
if (argv[i][0] == '-') {
fprintf(stderr, "unrecognized option: %s\n", argv[i]);
return EXIT_FAILURE;
}
size_t len;
uint8_t *src = readfile(argv[i], &len);
if (!src) {
fprintf(stderr, "error reading %s\n", argv[i]);
return EXIT_FAILURE;
}
/* convert UTF8 data to ICU's UTF16 */
UChar *usrc = (UChar*) malloc(2*len * sizeof(UChar));
ucnv_toUChars(uc, usrc, 2*len, (char*) src, len, &err);
if (U_FAILURE(err)) return EXIT_FAILURE;
size_t ulen = u_strlen(usrc);
/* ICU's insane normalization API requires you to
know the size of the destination buffer in advance,
or alternatively to repeatly try normalizing and
double the buffer size until it succeeds. Here, I just
allocate a huge destination buffer to avoid the issue. */
UChar *udest = (UChar*) malloc(10*ulen * sizeof(UChar));
mytime start = gettime();
for (int i = 0; i < 100; ++i) {
unorm2_normalize(NFKC, usrc, ulen, udest, 10*ulen, &err);
if (U_FAILURE(err)) return EXIT_FAILURE;
}
printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
free(udest);
free(usrc);
free(src);
}
return EXIT_SUCCESS;
}

60
lib/utf8proc-2a2f97e1/bench/unistring.c Normal file
View File

@ -0,0 +1,60 @@
/* comparitive benchmark of GNU libunistring */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/* libunistring */
#include <unistr.h>
#include <uninorm.h>
#include "util.h"
int main(int argc, char **argv)
{
int i;
uninorm_t nf = UNINORM_NFKC;
for (i = 1; i < argc; ++i) {
if (!strcmp(argv[i], "-nfkc")) {
nf = UNINORM_NFKC;
continue;
}
if (!strcmp(argv[i], "-nfkd")) {
nf = UNINORM_NFKD;
continue;
}
if (!strcmp(argv[i], "-nfc")) {
nf = UNINORM_NFC;
continue;
}
if (!strcmp(argv[i], "-nfd")) {
nf = UNINORM_NFD;
continue;
}
if (argv[i][0] == '-') {
fprintf(stderr, "unrecognized option: %s\n", argv[i]);
return EXIT_FAILURE;
}
size_t len;
uint8_t *src = readfile(argv[i], &len);
if (!src) {
fprintf(stderr, "error reading %s\n", argv[i]);
return EXIT_FAILURE;
}
size_t destlen;
uint8_t *dest;
mytime start = gettime();
for (int i = 0; i < 100; ++i) {
dest = u8_normalize(nf, src, len, NULL, &destlen);
if (!dest) return EXIT_FAILURE;
free(dest);
}
printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
free(src);
}
return EXIT_SUCCESS;
}

39
lib/utf8proc-2a2f97e1/bench/util.c Normal file
View File

@ -0,0 +1,39 @@
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include "util.h"
/* read file named FILENAME into an array of *len bytes,
returning NULL on error */
uint8_t *readfile(const char *filename, size_t *len)
{
*len = 0;
struct stat st;
if (0 != stat(filename, &st)) return NULL;
*len = st.st_size;
FILE *f = fopen(filename, "r");
if (!f) return NULL;
uint8_t *s = (uint8_t *) malloc(sizeof(uint8_t) * *len);
if (!s) return NULL;
if (fread(s, 1, *len, f) != *len) {
free(s);
s = NULL;
}
fclose(f);
return s;
}
mytime gettime(void) {
mytime t;
gettimeofday(&t, NULL);
return t;
}
/* time difference in seconds */
double elapsed(mytime t1, mytime t0)
{
return (double)(t1.tv_sec - t0.tv_sec) +
(double)(t1.tv_usec - t0.tv_usec) * 1.0E-6;
}

22
lib/utf8proc-2a2f97e1/bench/util.h Normal file
View File

@ -0,0 +1,22 @@
#ifndef UTIL_H
#define UTIL_H 1
#include <inttypes.h>
#include <sys/time.h>
#include <time.h>
#ifdef __cplusplus
extern "C" {
#endif
uint8_t *readfile(const char *filename, size_t *len);
typedef struct timeval mytime;
mytime gettime(void);
double elapsed(mytime t1, mytime t0);
#ifdef __cplusplus
}
#endif
#endif /* UTIL_H */

66
lib/utf8proc-2a2f97e1/data/Makefile Normal file
View File

@ -0,0 +1,66 @@
# Unicode data generation rules. Except for the test data files, most
# users will not use these Makefile rules, which are primarily to re-generate
# unicode_data.c when we get a new Unicode version or charwidth data; they
# require ruby, fontforge, and julia to be installed.
# programs
CURL=curl
RUBY=ruby
PERL=perl
MAKE=make
JULIA=julia
FONTFORGE=fontforge
CURLFLAGS = --retry 5 --location
.PHONY: clean
.DELETE_ON_ERROR:
utf8proc_data.c.new: data_generator.rb UnicodeData.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt CharWidths.txt
$(RUBY) data_generator.rb < UnicodeData.txt > $@
# GNU Unifont version for font metric calculations:
UNIFONT_VERSION=9.0.04
unifont.ttf:
$(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont-$(UNIFONT_VERSION).ttf
unifont_upper.ttf:
$(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont_upper-$(UNIFONT_VERSION).ttf
%.sfd: %.ttf
$(FONTFORGE) -lang=ff -c "Open(\"$<\");Save(\"$@\");Quit(0);"
CharWidths.txt: charwidths.jl unifont.sfd unifont_upper.sfd EastAsianWidth.txt
$(JULIA) charwidths.jl > $@
# Unicode data version
UNICODE_VERSION=9.0.0
UnicodeData.txt:
$(CURL) $(CURLFLAGS) -o $@ -O http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/UnicodeData.txt
EastAsianWidth.txt:
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/EastAsianWidth.txt
GraphemeBreakProperty.txt:
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/auxiliary/GraphemeBreakProperty.txt
DerivedCoreProperties.txt:
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/DerivedCoreProperties.txt
CompositionExclusions.txt:
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/CompositionExclusions.txt
CaseFolding.txt:
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/CaseFolding.txt
NormalizationTest.txt:
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/NormalizationTest.txt
GraphemeBreakTest.txt:
$(CURL) $(CURLFLAGS) $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/auxiliary/GraphemeBreakTest.txt | $(PERL) -pe 's,÷,/,g;s,×,+,g' > $@
clean:
rm -f UnicodeData.txt EastAsianWidth.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt NormalizationTest.txt GraphemeBreakTest.txt CharWidths.txt unifont*.ttf unifont*.sfd
rm -f utf8proc_data.c.new

190
lib/utf8proc-2a2f97e1/data/charwidths.jl Normal file
View File

@ -0,0 +1,190 @@
# Following work by @jiahao, we compute character widths using a combination of
# * advance widths from GNU Unifont (advance width 512 = 1 en)
# * UAX 11: East Asian Width
# * a few exceptions as needed
# Adapted from http://nbviewer.ipython.org/gist/jiahao/07e8b08bf6d8671e9734
#
# Requires Julia (obviously) and FontForge.
#############################################################################
# Julia 0.3/0.4 compatibility (taken from Compat package)
if VERSION < v"0.4.0-dev+1387"
typealias AbstractString String
end
if VERSION < v"0.4.0-dev+1419"
const UInt32 = Uint32
end
if VERSION < v"0.4.0-dev+3874"
Base.parse{T<:Integer}(::Type{T}, s::AbstractString) = parseint(T, s)
end
CharWidths = Dict{Int,Int}()
#############################################################################
# Use ../libutf8proc for category codes, rather than the one in Julia,
# to minimize bootstrapping complexity when a new version of Unicode comes out.
catcode(c) = ccall((:utf8proc_category,"../libutf8proc"), Cint, (Int32,), c)
# use Base.UTF8proc module to get category codes constants, since
# we won't change these in utf8proc.
import Base.UTF8proc
#############################################################################
# Use a default width of 1 for all character categories that are
# letter/symbol/number-like. This can be overriden by Unifont or UAX 11
# below, but provides a useful nonzero fallback for new codepoints when
# a new Unicode version has been released but Unifont hasn't been updated yet.
zerowidth = Set{Int}() # categories that may contain zero-width chars
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CN)
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_MN)
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_MC)
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ME)
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_SK)
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZS)
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZL)
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZP)
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CC)
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CF)
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CS)
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CO)
for c in 0x0000:0x110000
if catcode(c) ∉ zerowidth
CharWidths[c] = 1
end
end
#############################################################################
# Widths from GNU Unifont
#Read sfdfile for character widths
function parsesfd(filename::AbstractString, CharWidths::Dict{Int,Int}=Dict{Int,Int}())
state=:seekchar
lineno = 0
codepoint = width = nothing
for line in readlines(open(filename))
lineno += 1
if state==:seekchar #StartChar: nonmarkingreturn
if contains(line, "StartChar: ")
codepoint = nothing
width = nothing
state = :readdata
end
elseif state==:readdata #Encoding: 65538 -1 2, Width: 1024
contains(line, "Encoding:") && (codepoint = parse(Int, split(line)[3]))
contains(line, "Width:") && (width = parse(Int, split(line)[2]))
if codepoint!=nothing && width!=nothing && codepoint >= 0
w=div(width, 512) # 512 units to the en
if w > 0
# only add nonzero widths, since (1) the default is zero
# and (2) this circumvents some apparent bugs in Unifont
# (https://savannah.gnu.org/bugs/index.php?45395)
CharWidths[codepoint] = w
end
state = :seekchar
end
end
end
CharWidths
end
CharWidths=parsesfd("unifont.sfd", CharWidths)
CharWidths=parsesfd("unifont_upper.sfd", CharWidths)
#############################################################################
# Widths from UAX #11: East Asian Width
# .. these take precedence over the Unifont width for all codepoints
# listed explicitly as wide/full/narrow/half-width
for line in readlines(open("EastAsianWidth.txt"))
#Strip comments
line[1] == '#' && continue
precomment = split(line, '#')[1]
#Parse code point range and width code
tokens = split(precomment, ';')
length(tokens) >= 2 || continue
charrange = tokens[1]
width = strip(tokens[2])
#Parse code point range into Julia UnitRange
rangetokens = split(charrange, "..")
charstart = parse(UInt32, "0x"*rangetokens[1])
charend = parse(UInt32, "0x"*rangetokens[length(rangetokens)>1 ? 2 : 1])
#Assign widths
for c in charstart:charend
if width=="W" || width=="F" # wide or full
CharWidths[c]=2
elseif width=="Na"|| width=="H" # narrow or half
CharWidths[c]=1
end
end
end
#############################################################################
# A few exceptions to the above cases, found by manual comparison
# to other wcwidth functions and similar checks.
for c in keys(CharWidths)
cat = catcode(c)
# make sure format control character (category Cf) have width 0,
# except for the Arabic characters 0x06xx (see unicode std 6.2, sec. 8.2)
if cat==UTF8proc.UTF8PROC_CATEGORY_CF && c ∉ [0x0601,0x0602,0x0603,0x06dd]
CharWidths[c]=0
end
# Unifont has nonzero width for a number of non-spacing combining
# characters, e.g. (in 7.0.06): f84,17b4,17b5,180b,180d,2d7f, and
# the variation selectors
if cat==UTF8proc.UTF8PROC_CATEGORY_MN
CharWidths[c]=0
end
# We also assign width of zero to unassigned and private-use
# codepoints (Unifont includes ConScript Unicode Registry PUA fonts,
# but since these are nonstandard it seems questionable to recognize them).
if cat==UTF8proc.UTF8PROC_CATEGORY_CO || cat==UTF8proc.UTF8PROC_CATEGORY_CN
CharWidths[c]=0
end
# for some reason, Unifont has width-2 glyphs for ASCII control chars
if cat==UTF8proc.UTF8PROC_CATEGORY_CC
CharWidths[c]=0
end
end
#By definition, should have zero width (on the same line)
#0x002028 '
' category: Zl name: LINE SEPARATOR/
#0x002029 '
' category: Zp name: PARAGRAPH SEPARATOR/
CharWidths[0x2028]=0
CharWidths[0x2029]=0
#By definition, should be narrow = width of 1 en space
#0x00202f ' ' category: Zs name: NARROW NO-BREAK SPACE/
CharWidths[0x202f]=1
#By definition, should be wide = width of 1 em space
#0x002001 ' ' category: Zs name: EM QUAD/
#0x002003 ' ' category: Zs name: EM SPACE/
CharWidths[0x2001]=2
CharWidths[0x2003]=2
#############################################################################
# Output (to a file or pipe) for processing by data_generator.rb
# ... don't bother to output zero widths since that will be the default.
firstc = 0x000000
lastv = 0
uhex(c) = uppercase(hex(c,4))
for c in 0x0000:0x110000
v = get(CharWidths, c, 0)
if v != lastv || c == 0x110000
v < 4 || error("invalid charwidth $v for $c")
if firstc+1 < c
println(uhex(firstc), "..", uhex(c-1), "; ", lastv)
else
println(uhex(firstc), "; ", lastv)
end
firstc = c
lastv = v
end
end

411
lib/utf8proc-2a2f97e1/data/data_generator.rb Normal file
View File

@ -0,0 +1,411 @@
#!/usr/bin/env ruby
# This file was used to generate the 'unicode_data.c' file by parsing the
# Unicode data file 'UnicodeData.txt' of the Unicode Character Database.
# It is included for informational purposes only and not intended for
# production use.
# Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# This file contains derived data from a modified version of the
# Unicode data files. The following license applies to that data:
#
# COPYRIGHT AND PERMISSION NOTICE
#
# Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
# under the Terms of Use in http://www.unicode.org/copyright.html.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of the Unicode data files and any associated documentation (the "Data
# Files") or Unicode software and any associated documentation (the
# "Software") to deal in the Data Files or Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, and/or sell copies of the Data Files or Software, and
# to permit persons to whom the Data Files or Software are furnished to do
# so, provided that (a) the above copyright notice(s) and this permission
# notice appear with all copies of the Data Files or Software, (b) both the
# above copyright notice(s) and this permission notice appear in associated
# documentation, and (c) there is clear notice in each modified Data File or
# in the Software as well as in the documentation associated with the Data
# File(s) or Software that the data or software has been modified.
#
# THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
# THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
# INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR
# CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
# USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THE DATA FILES OR SOFTWARE.
#
# Except as contained in this notice, the name of a copyright holder shall
# not be used in advertising or otherwise to promote the sale, use or other
# dealings in these Data Files or Software without prior written
# authorization of the copyright holder.
$ignorable_list = File.read("DerivedCoreProperties.txt")[/# Derived Property: Default_Ignorable_Code_Point.*?# Total code points:/m]
$ignorable = []
$ignorable_list.each_line do |entry|
if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)/
$1.hex.upto($2.hex) { |e2| $ignorable << e2 }
elsif entry =~ /^[0-9A-F]+/
$ignorable << $&.hex
end
end
$grapheme_boundclass_list = File.read("GraphemeBreakProperty.txt")
$grapheme_boundclass = Hash.new("UTF8PROC_BOUNDCLASS_OTHER")
$grapheme_boundclass_list.each_line do |entry|
if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*([A-Za-z_]+)/
$1.hex.upto($2.hex) { |e2| $grapheme_boundclass[e2] = "UTF8PROC_BOUNDCLASS_" + $3.upcase }
elsif entry =~ /^([0-9A-F]+)\s*;\s*([A-Za-z_]+)/
$grapheme_boundclass[$1.hex] = "UTF8PROC_BOUNDCLASS_" + $2.upcase
end
end
$charwidth_list = File.read("CharWidths.txt")
$charwidth = Hash.new(0)
$charwidth_list.each_line do |entry|
if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*([0-9]+)/
$1.hex.upto($2.hex) { |e2| $charwidth[e2] = $3.to_i }
elsif entry =~ /^([0-9A-F]+)\s*;\s*([0-9]+)/
$charwidth[$1.hex] = $2.to_i
end
end
$exclusions = File.read("CompositionExclusions.txt")[/# \(1\) Script Specifics.*?# Total code points:/m]
$exclusions = $exclusions.chomp.split("\n").collect { |e| e.hex }
$excl_version = File.read("CompositionExclusions.txt")[/# \(2\) Post Composition Version precomposed characters.*?# Total code points:/m]
$excl_version = $excl_version.chomp.split("\n").collect { |e| e.hex }
$case_folding_string = File.open("CaseFolding.txt", :encoding => 'utf-8').read
$case_folding = {}
$case_folding_string.chomp.split("\n").each do |line|
next unless line =~ /([0-9A-F]+); [CFS]; ([0-9A-F ]+);/i
$case_folding[$1.hex] = $2.split(" ").collect { |e| e.hex }
end
$int_array = []
$int_array_indicies = {}
def str2c(string, prefix)
return "0" if string.nil?
return "UTF8PROC_#{prefix}_#{string.upcase}"
end
def pushary(array)
idx = $int_array_indicies[array]
unless idx
$int_array_indicies[array] = $int_array.length
idx = $int_array.length
array.each { |entry| $int_array << entry }
end
return idx
end
def cpary2utf16encoded(array)
return array.flat_map { |cp|
if (cp <= 0xFFFF)
raise "utf-16 code: #{cp}" if cp & 0b1111100000000000 == 0b1101100000000000
cp
else
temp = cp - 0x10000
[(temp >> 10) | 0b1101100000000000, (temp & 0b0000001111111111) | 0b1101110000000000]
end
}
end
def cpary2c(array)
return "UINT16_MAX" if array.nil? || array.length == 0
lencode = array.length - 1 #no sequence has len 0, so we encode len 1 as 0, len 2 as 1, ...
array = cpary2utf16encoded(array)
if lencode >= 7 #we have only 3 bits for the length (which is already cutting it close. might need to change it to 2 bits in future Unicode versions)
array = [lencode] + array
lencode = 7
end
idx = pushary(array)
raise "Array index out of bound" if idx > 0x1FFF
return "#{idx | (lencode << 13)}"
end
def singlecpmap(cp)
return "UINT16_MAX" if cp == nil
idx = pushary(cpary2utf16encoded([cp]))
raise "Array index out of bound" if idx > 0xFFFF
return "#{idx}"
end
class UnicodeChar
attr_accessor :code, :name, :category, :combining_class, :bidi_class,
:decomp_type, :decomp_mapping,
:bidi_mirrored,
:uppercase_mapping, :lowercase_mapping, :titlecase_mapping,
#caches:
:c_entry_index, :c_decomp_mapping, :c_case_folding
def initialize(line)
raise "Could not parse input." unless line =~ /^
([0-9A-F]+); # code
([^;]+); # name
([A-Z]+); # general category
([0-9]+); # canonical combining class
([A-Z]+); # bidi class
(<([A-Z]*)>)? # decomposition type
((\ ?[0-9A-F]+)*); # decompomposition mapping
([0-9]*); # decimal digit
([0-9]*); # digit
([^;]*); # numeric
([YN]*); # bidi mirrored
([^;]*); # unicode 1.0 name
([^;]*); # iso comment
([0-9A-F]*); # simple uppercase mapping
([0-9A-F]*); # simple lowercase mapping
([0-9A-F]*)$/ix # simple titlecase mapping
@code = $1.hex
@name = $2
@category = $3
@combining_class = Integer($4)
@bidi_class = $5
@decomp_type = $7
@decomp_mapping = ($8=='') ? nil :
$8.split.collect { |element| element.hex }
@bidi_mirrored = ($13=='Y') ? true : false
@uppercase_mapping = ($16=='') ? nil : $16.hex
@lowercase_mapping = ($17=='') ? nil : $17.hex
@titlecase_mapping = ($18=='') ? nil : $18.hex
end
def case_folding
$case_folding[code]
end
def c_entry(comb_indicies)
" " <<
"{#{str2c category, 'CATEGORY'}, #{combining_class}, " <<
"#{str2c bidi_class, 'BIDI_CLASS'}, " <<
"#{str2c decomp_type, 'DECOMP_TYPE'}, " <<
"#{c_decomp_mapping}, " <<
"#{c_case_folding}, " <<
"#{singlecpmap uppercase_mapping }, " <<
"#{singlecpmap lowercase_mapping }, " <<
"#{singlecpmap titlecase_mapping }, " <<
"#{comb_indicies[code] ? comb_indicies[code]: 'UINT16_MAX'}, " <<
"#{bidi_mirrored}, " <<
"#{$exclusions.include?(code) or $excl_version.include?(code)}, " <<
"#{$ignorable.include?(code)}, " <<
"#{%W[Zl Zp Cc Cf].include?(category) and not [0x200C, 0x200D].include?(category)}, " <<
"#{$charwidth[code]}, 0, " <<
"#{$grapheme_boundclass[code]}},\n"
end
end
chars = []
char_hash = {}
while gets
if $_ =~ /^([0-9A-F]+);<[^;>,]+, First>;/i
first = $1.hex
gets
char = UnicodeChar.new($_)
raise "No last character of sequence found." unless
$_ =~ /^([0-9A-F]+);<([^;>,]+), Last>;/i
last = $1.hex
name = "<#{$2}>"
for i in first..last
char_clone = char.clone
char_clone.code = i
char_clone.name = name
char_hash[char_clone.code] = char_clone
chars << char_clone
end
else
char = UnicodeChar.new($_)
char_hash[char.code] = char
chars << char
end
end
comb1st_indicies = {}
comb2nd_indicies = {}
comb2nd_indicies_sorted_keys = []
comb2nd_indicies_nonbasic = {}
comb_array = []
chars.each do |char|
if !char.nil? and char.decomp_type.nil? and char.decomp_mapping and
char.decomp_mapping.length == 2 and !char_hash[char.decomp_mapping[0]].nil? and
char_hash[char.decomp_mapping[0]].combining_class == 0 and
not $exclusions.include?(char.code)
dm0 = char.decomp_mapping[0]
dm1 = char.decomp_mapping[1]
unless comb1st_indicies[dm0]
comb1st_indicies[dm0] = comb1st_indicies.keys.length
end
unless comb2nd_indicies[dm1]
comb2nd_indicies_sorted_keys << dm1
comb2nd_indicies[dm1] = comb2nd_indicies.keys.length
end
comb_array[comb1st_indicies[dm0]] ||= []
raise "Duplicate canonical mapping: #{char.code} #{dm0} #{dm1}" if comb_array[comb1st_indicies[dm0]][comb2nd_indicies[dm1]]
comb_array[comb1st_indicies[dm0]][comb2nd_indicies[dm1]] = char.code
comb2nd_indicies_nonbasic[dm1] = true if char.code > 0xFFFF
end
char.c_decomp_mapping = cpary2c(char.decomp_mapping)
char.c_case_folding = cpary2c(char.case_folding)
end
comb_indicies = {}
cumoffset = 0
comb1st_indicies_lastoffsets = []
comb1st_indicies_firstoffsets = []
comb1st_indicies.each do |dm0, index|
first = nil
last = nil
offset = 0
comb2nd_indicies_sorted_keys.each_with_index do |dm1, b|
if comb_array[index][b]
first = offset unless first
last = offset
last += 1 if comb2nd_indicies_nonbasic[dm1]
end
offset += 1
offset += 1 if comb2nd_indicies_nonbasic[dm1]
end
comb1st_indicies_firstoffsets[index] = first
comb1st_indicies_lastoffsets[index] = last
raise "double index" if comb_indicies[dm0]
comb_indicies[dm0] = cumoffset
cumoffset += last - first + 1 + 2
end
offset = 0
comb2nd_indicies_sorted_keys.each do |dm1|
raise "double index" if comb_indicies[dm1]
comb_indicies[dm1] = 0x8000 | (comb2nd_indicies[dm1] + offset)
raise "too large comb index" if comb2nd_indicies[dm1] + offset > 0x4000
if comb2nd_indicies_nonbasic[dm1]
comb_indicies[dm1] = comb_indicies[dm1] | 0x4000
offset += 1
end
end
properties_indicies = {}
properties = []
chars.each do |char|
c_entry = char.c_entry(comb_indicies)
char.c_entry_index = properties_indicies[c_entry]
unless char.c_entry_index
properties_indicies[c_entry] = properties.length
char.c_entry_index = properties.length
properties << c_entry
end
end
stage1 = []
stage2 = []
for code in 0...0x110000
next unless code % 0x100 == 0
stage2_entry = []
for code2 in code...(code+0x100)
if char_hash[code2]
stage2_entry << (char_hash[code2].c_entry_index + 1)
else
stage2_entry << 0
end
end
old_index = stage2.index(stage2_entry)
if old_index
stage1 << (old_index * 0x100)
else
stage1 << (stage2.length * 0x100)
stage2 << stage2_entry
end
end
$stdout << "static const utf8proc_uint16_t utf8proc_sequences[] = {\n "
i = 0
$int_array.each do |entry|
i += 1
if i == 8
i = 0
$stdout << "\n "
end
$stdout << entry << ", "
end
$stdout << "};\n\n"
$stdout << "static const utf8proc_uint16_t utf8proc_stage1table[] = {\n "
i = 0
stage1.each do |entry|
i += 1
if i == 8
i = 0
$stdout << "\n "
end
$stdout << entry << ", "
end
$stdout << "};\n\n"
$stdout << "static const utf8proc_uint16_t utf8proc_stage2table[] = {\n "
i = 0
stage2.flatten.each do |entry|
i += 1
if i == 8
i = 0
$stdout << "\n "
end
$stdout << entry << ", "
end
$stdout << "};\n\n"
$stdout << "static const utf8proc_property_t utf8proc_properties[] = {\n"
$stdout << " {0, 0, 0, 0, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, false,false,false,false, 0, 0, UTF8PROC_BOUNDCLASS_OTHER},\n"
properties.each { |line|
$stdout << line
}
$stdout << "};\n\n"
$stdout << "static const utf8proc_uint16_t utf8proc_combinations[] = {\n "
i = 0
comb1st_indicies.keys.each_index do |a|
offset = 0
$stdout << comb1st_indicies_firstoffsets[a] << ", " << comb1st_indicies_lastoffsets[a] << ", "
comb2nd_indicies_sorted_keys.each_with_index do |dm1, b|
break if offset > comb1st_indicies_lastoffsets[a]
if offset >= comb1st_indicies_firstoffsets[a]
i += 1
if i == 8
i = 0
$stdout << "\n "
end
v = comb_array[a][b] ? comb_array[a][b] : 0
$stdout << (( v & 0xFFFF0000 ) >> 16) << ", " if comb2nd_indicies_nonbasic[dm1]
$stdout << (v & 0xFFFF) << ", "
end
offset += 1
offset += 1 if comb2nd_indicies_nonbasic[dm1]
end
$stdout << "\n"
end
$stdout << "};\n\n"

27
lib/utf8proc-2a2f97e1/lump.md Normal file
View File

@ -0,0 +1,27 @@
```
U+0020 <-- all space characters (general category Zs)
U+0027 ' <-- left/right single quotation mark U+2018..2019,
modifier letter apostrophe U+02BC,
modifier letter vertical line U+02C8
U+002D - <-- all dash characters (general category Pd),
minus U+2212
U+002F / <-- fraction slash U+2044,
division slash U+2215
U+003A : <-- ratio U+2236
U+003C < <-- single left-pointing angle quotation mark U+2039,
left-pointing angle bracket U+2329,
left angle bracket U+3008
U+003E > <-- single right-pointing angle quotation mark U+203A,
right-pointing angle bracket U+232A,
right angle bracket U+3009
U+005C \ <-- set minus U+2216
U+005E ^ <-- modifier letter up arrowhead U+02C4,
modifier letter circumflex accent U+02C6,
caret U+2038,
up arrowhead U+2303
U+005F _ <-- all connector characters (general category Pc),
modifier letter low macron U+02CD
U+0060 ` <-- modifier letter grave accent U+02CB
U+007C | <-- divides U+2223
U+007E ~ <-- tilde operator U+223C
```

50
lib/utf8proc-2a2f97e1/test/case.c Normal file
View File

@ -0,0 +1,50 @@
#include "tests.h"
#include <wctype.h>
int main(int argc, char **argv)
{
int error = 0, better = 0;
utf8proc_int32_t c;
(void) argc; /* unused */
(void) argv; /* unused */
/* some simple sanity tests of the character widths */
for (c = 0; c <= 0x110000; ++c) {
utf8proc_int32_t l = utf8proc_tolower(c);
utf8proc_int32_t u = utf8proc_toupper(c);
check(l == c || utf8proc_codepoint_valid(l), "invalid tolower");
check(u == c || utf8proc_codepoint_valid(u), "invalid toupper");
if (sizeof(wint_t) > 2 || c < (1<<16)) {
wint_t l0 = towlower(c), u0 = towupper(c);
/* OS unicode tables may be out of date. But if they
do have a lower/uppercase mapping, hopefully it
is correct? */
if (l0 != c && l0 != l) {
fprintf(stderr, "MISMATCH %x != towlower(%x) == %x\n",
l, c, l0);
++error;
}
else if (l0 != l) { /* often true for out-of-date OS unicode */
++better;
/* printf("%x != towlower(%x) == %x\n", l, c, l0); */
}
if (u0 != c && u0 != u) {
fprintf(stderr, "MISMATCH %x != towupper(%x) == %x\n",
u, c, u0);
++error;
}
else if (u0 != u) { /* often true for out-of-date OS unicode */
++better;
/* printf("%x != towupper(%x) == %x\n", u, c, u0); */
}
}
}
check(!error, "utf8proc case conversion FAILED %d tests.", error);
printf("More up-to-date than OS unicode tables for %d tests.\n", better);
printf("utf8proc case conversion tests SUCCEEDED.\n");
return 0;
}

71
lib/utf8proc-2a2f97e1/test/charwidth.c Normal file
View File

@ -0,0 +1,71 @@
#include "tests.h"
#include <ctype.h>
#include <wchar.h>
static int my_isprint(int c) {
int cat = utf8proc_get_property(c)->category;
return (UTF8PROC_CATEGORY_LU <= cat && cat <= UTF8PROC_CATEGORY_ZS) ||
(c == 0x0601 || c == 0x0602 || c == 0x0603 || c == 0x06dd);
}
int main(int argc, char **argv)
{
int c, error = 0, updates = 0;
(void) argc; /* unused */
(void) argv; /* unused */
/* some simple sanity tests of the character widths */
for (c = 0; c <= 0x110000; ++c) {
int cat = utf8proc_get_property(c)->category;
int w = utf8proc_charwidth(c);
if ((cat == UTF8PROC_CATEGORY_MN || cat == UTF8PROC_CATEGORY_ME) &&
w > 0) {
fprintf(stderr, "nonzero width %d for combining char %x\n", w, c);
error = 1;
}
if (w == 0 &&
((cat >= UTF8PROC_CATEGORY_LU && cat <= UTF8PROC_CATEGORY_LO) ||
(cat >= UTF8PROC_CATEGORY_ND && cat <= UTF8PROC_CATEGORY_SC) ||
(cat >= UTF8PROC_CATEGORY_SO && cat <= UTF8PROC_CATEGORY_ZS))) {
fprintf(stderr, "zero width for symbol-like char %x\n", c);
error = 1;
}
if (c <= 127 && ((!isprint(c) && w > 0) ||
(isprint(c) && wcwidth(c) != w))) {
fprintf(stderr, "wcwidth %d mismatch %d for %s ASCII %x\n",
wcwidth(c), w,
isprint(c) ? "printable" : "non-printable", c);
error = 1;
}
if (!my_isprint(c) && w > 0) {
fprintf(stderr, "non-printing %x had width %d\n", c, w);
error = 1;
}
}
check(!error, "utf8proc_charwidth FAILED tests.");
/* print some other information by compariing with system wcwidth */
printf("Mismatches with system wcwidth (not necessarily errors):\n");
for (c = 0; c <= 0x110000; ++c) {
int w = utf8proc_charwidth(c);
int wc = wcwidth(c);
if (sizeof(wchar_t) == 2 && c >= (1<<16)) continue;
/* lots of these errors for out-of-date system unicode tables */
if (wc == -1 && my_isprint(c) && w > 0) {
updates += 1;
#if 0
printf(" wcwidth(%x) = -1 for printable char\n", c);
#endif
}
if (wc == -1 && !my_isprint(c) && w > 0)
printf(" wcwidth(%x) = -1 for non-printable width-%d char\n", c, w);
if (wc >= 0 && wc != w)
printf(" wcwidth(%x) = %d != charwidth %d\n", c, wc, w);
}
printf(" ... (positive widths for %d chars unknown to wcwidth) ...\n",
updates);
printf("Character-width tests SUCCEEDED.\n");
return 0;
}

27
lib/utf8proc-2a2f97e1/test/custom.c Normal file
View File

@ -0,0 +1,27 @@
#include "tests.h"
static int thunk_test = 1;
static utf8proc_int32_t custom(utf8proc_int32_t codepoint, void *thunk)
{
check(((int *) thunk) == &thunk_test, "unexpected thunk passed");
if (codepoint == 'a')
return 'b';
if (codepoint == 'S')
return 0x00df; /* ß */
return codepoint;
}
int main(void)
{
utf8proc_uint8_t input[] = {0x41,0x61,0x53,0x62,0xef,0xbd,0x81,0x00}; /* "AaSb\uff41" */
utf8proc_uint8_t correct[] = {0x61,0x62,0x73,0x73,0x62,0x61,0x00}; /* "abssba" */
utf8proc_uint8_t *output;
utf8proc_map_custom(input, 0, &output, UTF8PROC_CASEFOLD | UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_NULLTERM,
custom, &thunk_test);
printf("mapped \"%s\" -> \"%s\"\n", (char*)input, (char*)output);
check(strlen((char*) output) == 6, "incorrect output length");
check(!memcmp(correct, output, 7), "incorrect output data");
free(output);
return 0;
}

74
lib/utf8proc-2a2f97e1/test/graphemetest.c Normal file
View File

@ -0,0 +1,74 @@
#include "tests.h"
int main(int argc, char **argv)
{
char *buf = NULL;
size_t bufsize = 0;
FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
utf8proc_uint8_t src[1024];
int len;
check(f != NULL, "error opening GraphemeBreakTest.txt");
while (getline(&buf, &bufsize, f) > 0) {
size_t bi = 0, si = 0;
lineno += 1;
if (lineno % 100 == 0)
printf("checking line %zd...\n", lineno);
if (buf[0] == '#') continue;
while (buf[bi]) {
bi = skipspaces(buf, bi);
if (buf[bi] == '/') { /* grapheme break */
src[si++] = '/';
bi++;
}
else if (buf[bi] == '+') { /* no break */
bi++;
}
else if (buf[bi] == '#') { /* start of comments */
break;
}
else { /* hex-encoded codepoint */
len = encode((char*) (src + si), buf + bi) - 1;
while (src[si]) ++si; /* advance to NUL termination */
bi += len;
}
}
if (si && src[si-1] == '/')
--si; /* no break after final grapheme */
src[si] = 0; /* NUL-terminate */
if (si) {
utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */
size_t i = 0, j = 0;
utf8proc_ssize_t glen;
utf8proc_uint8_t *g; /* utf8proc_map grapheme results */
while (i < si) {
if (src[i] != '/')
utf8[j++] = src[i++];
else
i++;
}
glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND);
if (glen == UTF8PROC_ERROR_INVALIDUTF8) {
/* the test file contains surrogate codepoints, which are only for UTF-16 */
printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno);
}
else {
check(glen >= 0, "utf8proc_map error = %s",
utf8proc_errmsg(glen));
for (i = 0; i <= glen; ++i)
if (g[i] == 0xff)
g[i] = '/'; /* easier-to-read output (/ is not in test strings) */
check(!strcmp((char*)g, (char*)src),
"grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src);
}
free(g);
}
}
fclose(f);
printf("Passed tests after %zd lines!\n", lineno);
return 0;
}

166
lib/utf8proc-2a2f97e1/test/iterate.c Normal file
View File

@ -0,0 +1,166 @@
#include "tests.h"
#include <ctype.h>
#include <wchar.h>
static int tests;
static int error;
#define CHECKVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,len,__LINE__)
#define CHECKINVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,UTF8PROC_ERROR_INVALIDUTF8,__LINE__)
static void testbytes(unsigned char *buf, int len, utf8proc_ssize_t retval, int line)
{
utf8proc_int32_t out[16];
utf8proc_ssize_t ret;
/* Make a copy to ensure that memory is left uninitialized after "len"
* bytes. This way, Valgrind can detect overreads.
*/
unsigned char tmp[16];
memcpy(tmp, buf, len);
tests++;
if ((ret = utf8proc_iterate(tmp, len, out)) != retval) {
fprintf(stderr, "Failed (%d):", line);
for (int i = 0; i < len ; i++) {
fprintf(stderr, " 0x%02x", tmp[i]);
}
fprintf(stderr, " -> %zd\n", ret);
error++;
}
}
int main(int argc, char **argv)
{
uint32_t byt;
unsigned char buf[16];
tests = error = 0;
// Check valid sequences that were considered valid erroneously before
buf[0] = 0xef;
buf[1] = 0xb7;
for (byt = 0x90; byt < 0xa0; byt++) {
CHECKVALID(2, byt, 3);
}
// Check 0xfffe and 0xffff
buf[1] = 0xbf;
CHECKVALID(2, 0xbe, 3);
CHECKVALID(2, 0xbf, 3);
// Check 0x??fffe & 0x??ffff
for (byt = 0x1fffe; byt < 0x110000; byt += 0x10000) {
buf[0] = 0xf0 | (byt >> 18);
buf[1] = 0x80 | ((byt >> 12) & 0x3f);
CHECKVALID(3, 0xbe, 4);
CHECKVALID(3, 0xbf, 4);
}
// Continuation byte not after lead
for (byt = 0x80; byt < 0xc0; byt++) {
CHECKINVALID(0, byt, 1);
}
// Continuation byte not after lead
for (byt = 0x80; byt < 0xc0; byt++) {
CHECKINVALID(0, byt, 1);
}
// Test lead bytes
for (byt = 0xc0; byt <= 0xff; byt++) {
// Single lead byte at end of string
CHECKINVALID(0, byt, 1);
// Lead followed by non-continuation character < 0x80
CHECKINVALID(1, 65, 2);
// Lead followed by non-continuation character > 0xbf
CHECKINVALID(1, 0xc0, 2);
}
// Test overlong 2-byte
buf[0] = 0xc0;
for (byt = 0x81; byt <= 0xbf; byt++) {
CHECKINVALID(1, byt, 2);
}
buf[0] = 0xc1;
for (byt = 0x80; byt <= 0xbf; byt++) {
CHECKINVALID(1, byt, 2);
}
// Test overlong 3-byte
buf[0] = 0xe0;
buf[2] = 0x80;
for (byt = 0x80; byt <= 0x9f; byt++) {
CHECKINVALID(1, byt, 3);
}
// Test overlong 4-byte
buf[0] = 0xf0;
buf[2] = 0x80;
buf[3] = 0x80;
for (byt = 0x80; byt <= 0x8f; byt++) {
CHECKINVALID(1, byt, 4);
}
// Test 4-byte > 0x10ffff
buf[0] = 0xf4;
buf[2] = 0x80;
buf[3] = 0x80;
for (byt = 0x90; byt <= 0xbf; byt++) {
CHECKINVALID(1, byt, 4);
}
buf[1] = 0x80;
for (byt = 0xf5; byt <= 0xf7; byt++) {
CHECKINVALID(0, byt, 4);
}
// Test 5-byte
buf[4] = 0x80;
for (byt = 0xf8; byt <= 0xfb; byt++) {
CHECKINVALID(0, byt, 5);
}
// Test 6-byte
buf[5] = 0x80;
for (byt = 0xfc; byt <= 0xfd; byt++) {
CHECKINVALID(0, byt, 6);
}
// Test 7-byte
buf[6] = 0x80;
CHECKINVALID(0, 0xfe, 7);
// Three and above byte sequences
for (byt = 0xe0; byt < 0xf0; byt++) {
// Lead followed by only 1 continuation byte
CHECKINVALID(0, byt, 2);
// Lead ended by non-continuation character < 0x80
CHECKINVALID(2, 65, 3);
// Lead ended by non-continuation character > 0xbf
CHECKINVALID(2, 0xc0, 3);
}
// 3-byte encoded surrogate character(s)
buf[0] = 0xed; buf[2] = 0x80;
// Single surrogate
CHECKINVALID(1, 0xa0, 3);
// Trailing surrogate first
CHECKINVALID(1, 0xb0, 3);
// Four byte sequences
buf[1] = 0x80;
for (byt = 0xf0; byt < 0xf5; byt++) {
// Lead followed by only 1 continuation bytes
CHECKINVALID(0, byt, 2);
// Lead followed by only 2 continuation bytes
CHECKINVALID(0, byt, 3);
// Lead followed by non-continuation character < 0x80
CHECKINVALID(3, 65, 4);
// Lead followed by non-continuation character > 0xbf
CHECKINVALID(3, 0xc0, 4);
}
check(!error, "utf8proc_iterate FAILED %d tests out of %d", error, tests);
printf("utf8proc_iterate tests SUCCEEDED, (%d) tests passed.\n", tests);
return 0;
}

64
lib/utf8proc-2a2f97e1/test/normtest.c Normal file
View File

@ -0,0 +1,64 @@
#include "tests.h"
#define CHECK_NORM(NRM, norm, src) { \
char *src_norm = (char*) utf8proc_ ## NRM((utf8proc_uint8_t*) src); \
check(!strcmp(norm, src_norm), \
"normalization failed for %s -> %s", src, norm); \
free(src_norm); \
}
int main(int argc, char **argv)
{
char *buf = NULL;
size_t bufsize = 0;
FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024];
check(f != NULL, "error opening NormalizationTest.txt");
while (getline(&buf, &bufsize, f) > 0) {
size_t offset;
lineno += 1;
if (buf[0] == '@') {
printf("line %zd: %s", lineno, buf + 1);
continue;
}
else if (lineno % 1000 == 0)
printf("checking line %zd...\n", lineno);
if (buf[0] == '#') continue;
offset = encode(source, buf);
offset += encode(NFC, buf + offset);
offset += encode(NFD, buf + offset);
offset += encode(NFKC, buf + offset);
offset += encode(NFKD, buf + offset);
CHECK_NORM(NFC, NFC, source);
CHECK_NORM(NFC, NFC, NFC);
CHECK_NORM(NFC, NFC, NFD);
CHECK_NORM(NFC, NFKC, NFKC);
CHECK_NORM(NFC, NFKC, NFKD);
CHECK_NORM(NFD, NFD, source);
CHECK_NORM(NFD, NFD, NFC);
CHECK_NORM(NFD, NFD, NFD);
CHECK_NORM(NFD, NFKD, NFKC);
CHECK_NORM(NFD, NFKD, NFKD);
CHECK_NORM(NFKC, NFKC, source);
CHECK_NORM(NFKC, NFKC, NFC);
CHECK_NORM(NFKC, NFKC, NFD);
CHECK_NORM(NFKC, NFKC, NFKC);
CHECK_NORM(NFKC, NFKC, NFKD);
CHECK_NORM(NFKD, NFKD, source);
CHECK_NORM(NFKD, NFKD, NFC);
CHECK_NORM(NFKD, NFKD, NFD);
CHECK_NORM(NFKD, NFKD, NFKC);
CHECK_NORM(NFKD, NFKD, NFKD);
}
fclose(f);
printf("Passed tests after %zd lines!\n", lineno);
return 0;
}

49
lib/utf8proc-2a2f97e1/test/printproperty.c Normal file
View File

@ -0,0 +1,49 @@
/* simple test program to print out the utf8proc properties for a codepoint */
#include "tests.h"
int main(int argc, char **argv)
{
int i;
for (i = 1; i < argc; ++i) {
unsigned int c;
if (!strcmp(argv[i], "-V")) {
printf("utf8proc version %s\n", utf8proc_version());
continue;
}
check(sscanf(argv[i],"%x",&c) == 1, "invalid hex input %s", argv[i]);
const utf8proc_property_t *p = utf8proc_get_property(c);
printf("U+%s:\n"
" category = %s\n"
" combining_class = %d\n"
" bidi_class = %d\n"
" decomp_type = %d\n"
" uppercase_mapping = %x\n"
" lowercase_mapping = %x\n"
" titlecase_mapping = %x\n"
" comb_index = %d\n"
" bidi_mirrored = %d\n"
" comp_exclusion = %d\n"
" ignorable = %d\n"
" control_boundary = %d\n"
" boundclass = %d\n"
" charwidth = %d\n",
argv[i],
utf8proc_category_string(c),
p->combining_class,
p->bidi_class,
p->decomp_type,
utf8proc_toupper(c),
utf8proc_tolower(c),
utf8proc_totitle(c),
p->comb_index,
p->bidi_mirrored,
p->comp_exclusion,
p->ignorable,
p->control_boundary,
p->boundclass,
utf8proc_charwidth(c));
}
return 0;
}

46
lib/utf8proc-2a2f97e1/test/tests.c Normal file
View File

@ -0,0 +1,46 @@
/* Common functions for our test programs. */
#include "tests.h"
size_t lineno = 0;
void check(int cond, const char *format, ...)
{
if (!cond) {
va_list args;
fprintf(stderr, "line %zd: ", lineno);
va_start(args, format);
vfprintf(stderr, format, args);
va_end(args);
fprintf(stderr, "\n");
exit(1);
}
}
size_t skipspaces(const char *buf, size_t i)
{
while (isspace(buf[i])) ++i;
return i;
}
/* if buf points to a sequence of codepoints encoded as hexadecimal strings,
separated by whitespace, and terminated by any character not in
[0-9a-fA-F] or whitespace, then stores the corresponding utf8 string
in dest, returning the number of bytes read from buf */
size_t encode(char *dest, const char *buf)
{
size_t i = 0, j, d = 0;
for (;;) {
int c;
i = skipspaces(buf, i);
for (j=i; buf[j] && strchr("0123456789abcdef", tolower(buf[j])); ++j)
; /* find end of hex input */
if (j == i) { /* no codepoint found */
dest[d] = 0; /* NUL-terminate destination string */
return i + 1;
}
check(sscanf(buf + i, "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i);
i = j; /* skip to char after hex input */
d += utf8proc_encode_char(c, (utf8proc_uint8_t *) (dest + d));
}
}

23
lib/utf8proc-2a2f97e1/test/tests.h Normal file
View File

@ -0,0 +1,23 @@
/* Common functions and includes for our test programs. */
/*
* Set feature macro to enable getline() and wcwidth().
*
* Please refer to section 2.2.1 of POSIX.1-2008:
* http://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_02_01_02
*/
#define _XOPEN_SOURCE 700
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <string.h>
#include <stdarg.h>
#include "../utf8proc.h"
extern size_t lineno;
void check(int cond, const char *format, ...);
size_t skipspaces(const char *buf, size_t i);
size_t encode(char *dest, const char *buf);

41
lib/utf8proc-2a2f97e1/test/valid.c Normal file
View File

@ -0,0 +1,41 @@
#include "tests.h"
#include <ctype.h>
#include <wchar.h>
int main(int argc, char **argv)
{
int c, error = 0;
(void) argc; /* unused */
(void) argv; /* unused */
/* some simple sanity tests of */
for (c = 0; c < 0xd800; c++) {
if (!utf8proc_codepoint_valid(c)) {
fprintf(stderr, "Failed: codepoint_valid(%04x) -> false\n", c);
error++;
}
}
for (;c < 0xe000; c++) {
if (utf8proc_codepoint_valid(c)) {
fprintf(stderr, "Failed: codepoint_valid(%04x) -> true\n", c);
error++;
}
}
for (;c < 0x110000; c++) {
if (!utf8proc_codepoint_valid(c)) {
fprintf(stderr, "Failed: codepoint_valid(%06x) -> false\n", c);
error++;
}
}
for (;c < 0x110010; c++) {
if (utf8proc_codepoint_valid(c)) {
fprintf(stderr, "Failed: codepoint_valid(%06x) -> true\n", c);
error++;
}
}
check(!error, "utf8proc_codepoint_valid FAILED %d tests.", error);
printf("Validity tests SUCCEEDED.\n");
return 0;
}

755
lib/utf8proc-2a2f97e1/utf8proc.c Normal file
View File

@ -0,0 +1,755 @@
/* -*- mode: c; c-basic-offset: 2; tab-width: 2; indent-tabs-mode: nil -*- */
/*
* Copyright (c) 2015 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
* Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
/*
* This library contains derived data from a modified version of the
* Unicode data files.
*
* The original data files are available at
* http://www.unicode.org/Public/UNIDATA/
*
* Please notice the copyright statement in the file "utf8proc_data.c".
*/
/*
* File name: utf8proc.c
*
* Description:
* Implementation of libutf8proc.
*/
#include "utf8proc.h"
#include "utf8proc_data.c"
UTF8PROC_DLLEXPORT const utf8proc_int8_t utf8proc_utf8class[256] = {
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0 };
#define UTF8PROC_HANGUL_SBASE 0xAC00
#define UTF8PROC_HANGUL_LBASE 0x1100
#define UTF8PROC_HANGUL_VBASE 0x1161
#define UTF8PROC_HANGUL_TBASE 0x11A7
#define UTF8PROC_HANGUL_LCOUNT 19
#define UTF8PROC_HANGUL_VCOUNT 21
#define UTF8PROC_HANGUL_TCOUNT 28
#define UTF8PROC_HANGUL_NCOUNT 588
#define UTF8PROC_HANGUL_SCOUNT 11172
/* END is exclusive */
#define UTF8PROC_HANGUL_L_START 0x1100
#define UTF8PROC_HANGUL_L_END 0x115A
#define UTF8PROC_HANGUL_L_FILLER 0x115F
#define UTF8PROC_HANGUL_V_START 0x1160
#define UTF8PROC_HANGUL_V_END 0x11A3
#define UTF8PROC_HANGUL_T_START 0x11A8
#define UTF8PROC_HANGUL_T_END 0x11FA
#define UTF8PROC_HANGUL_S_START 0xAC00
#define UTF8PROC_HANGUL_S_END 0xD7A4
/* Should follow semantic-versioning rules (semver.org) based on API
compatibility. (Note that the shared-library version number will
be different, being based on ABI compatibility.): */
#define STRINGIZEx(x) #x
#define STRINGIZE(x) STRINGIZEx(x)
UTF8PROC_DLLEXPORT const char *utf8proc_version(void) {
return STRINGIZE(UTF8PROC_VERSION_MAJOR) "." STRINGIZE(UTF8PROC_VERSION_MINOR) "." STRINGIZE(UTF8PROC_VERSION_PATCH) "";
}
UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode) {
switch (errcode) {
case UTF8PROC_ERROR_NOMEM:
return "Memory for processing UTF-8 data could not be allocated.";
case UTF8PROC_ERROR_OVERFLOW:
return "UTF-8 string is too long to be processed.";
case UTF8PROC_ERROR_INVALIDUTF8:
return "Invalid UTF-8 string";
case UTF8PROC_ERROR_NOTASSIGNED:
return "Unassigned Unicode code point found in UTF-8 string.";
case UTF8PROC_ERROR_INVALIDOPTS:
return "Invalid options for UTF-8 processing chosen.";
default:
return "An unknown error occurred while processing UTF-8 data.";
}
}
#define utf_cont(ch) (((ch) & 0xc0) == 0x80)
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *dst
) {
utf8proc_uint32_t uc;
const utf8proc_uint8_t *end;
*dst = -1;
if (!strlen) return 0;
end = str + ((strlen < 0) ? 4 : strlen);
uc = *str++;
if (uc < 0x80) {
*dst = uc;
return 1;
}
// Must be between 0xc2 and 0xf4 inclusive to be valid
if ((uc - 0xc2) > (0xf4-0xc2)) return UTF8PROC_ERROR_INVALIDUTF8;
if (uc < 0xe0) { // 2-byte sequence
// Must have valid continuation character
if (str >= end || !utf_cont(*str)) return UTF8PROC_ERROR_INVALIDUTF8;
*dst = ((uc & 0x1f)<<6) | (*str & 0x3f);
return 2;
}
if (uc < 0xf0) { // 3-byte sequence
if ((str + 1 >= end) || !utf_cont(*str) || !utf_cont(str[1]))
return UTF8PROC_ERROR_INVALIDUTF8;
// Check for surrogate chars
if (uc == 0xed && *str > 0x9f)
return UTF8PROC_ERROR_INVALIDUTF8;
uc = ((uc & 0xf)<<12) | ((*str & 0x3f)<<6) | (str[1] & 0x3f);
if (uc < 0x800)
return UTF8PROC_ERROR_INVALIDUTF8;
*dst = uc;
return 3;
}
// 4-byte sequence
// Must have 3 valid continuation characters
if ((str + 2 >= end) || !utf_cont(*str) || !utf_cont(str[1]) || !utf_cont(str[2]))
return UTF8PROC_ERROR_INVALIDUTF8;
// Make sure in correct range (0x10000 - 0x10ffff)
if (uc == 0xf0) {
if (*str < 0x90) return UTF8PROC_ERROR_INVALIDUTF8;
} else if (uc == 0xf4) {
if (*str > 0x8f) return UTF8PROC_ERROR_INVALIDUTF8;
}
*dst = ((uc & 7)<<18) | ((*str & 0x3f)<<12) | ((str[1] & 0x3f)<<6) | (str[2] & 0x3f);
return 4;
}
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t uc) {
return (((utf8proc_uint32_t)uc)-0xd800 > 0x07ff) && ((utf8proc_uint32_t)uc < 0x110000);
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
if (uc < 0x00) {
return 0;
} else if (uc < 0x80) {
dst[0] = (utf8proc_uint8_t) uc;
return 1;
} else if (uc < 0x800) {
dst[0] = (utf8proc_uint8_t)(0xC0 + (uc >> 6));
dst[1] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 2;
// Note: we allow encoding 0xd800-0xdfff here, so as not to change
// the API, however, these are actually invalid in UTF-8
} else if (uc < 0x10000) {
dst[0] = (utf8proc_uint8_t)(0xE0 + (uc >> 12));
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
dst[2] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 3;
} else if (uc < 0x110000) {
dst[0] = (utf8proc_uint8_t)(0xF0 + (uc >> 18));
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 12) & 0x3F));
dst[2] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
dst[3] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 4;
} else return 0;
}
/* internal "unsafe" version that does not check whether uc is in range */
static utf8proc_ssize_t unsafe_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
if (uc < 0x00) {
return 0;
} else if (uc < 0x80) {
dst[0] = (utf8proc_uint8_t)uc;
return 1;
} else if (uc < 0x800) {
dst[0] = (utf8proc_uint8_t)(0xC0 + (uc >> 6));
dst[1] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 2;
} else if (uc == 0xFFFF) {
dst[0] = (utf8proc_uint8_t)0xFF;
return 1;
} else if (uc == 0xFFFE) {
dst[0] = (utf8proc_uint8_t)0xFE;
return 1;
} else if (uc < 0x10000) {
dst[0] = (utf8proc_uint8_t)(0xE0 + (uc >> 12));
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
dst[2] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 3;
} else if (uc < 0x110000) {
dst[0] = (utf8proc_uint8_t)(0xF0 + (uc >> 18));
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 12) & 0x3F));
dst[2] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
dst[3] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
return 4;
} else return 0;
}
/* internal "unsafe" version that does not check whether uc is in range */
static const utf8proc_property_t *unsafe_get_property(utf8proc_int32_t uc) {
/* ASSERT: uc >= 0 && uc < 0x110000 */
return utf8proc_properties + (
utf8proc_stage2table[
utf8proc_stage1table[uc >> 8] + (uc & 0xFF)
]
);
}
UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t uc) {
return uc < 0 || uc >= 0x110000 ? utf8proc_properties : unsafe_get_property(uc);
}
/* return whether there is a grapheme break between boundclasses lbc and tbc
(according to the definition of extended grapheme clusters)
Rule numbering refers to TR29 Version 29 (Unicode 9.0.0):
http://www.unicode.org/reports/tr29/tr29-29.html
CAVEATS:
Please note that evaluation of GB10 (grapheme breaks between emoji zwj sequences)
and GB 12/13 (regional indicator code points) require knowledge of previous characters
and are thus not handled by this function. This may result in an incorrect break before
an E_Modifier class codepoint and an incorrectly missing break between two
REGIONAL_INDICATOR class code points if such support does not exist in the caller.
See the special support in grapheme_break_extended, for required bookkeeping by the caller.
*/
static utf8proc_bool grapheme_break_simple(int lbc, int tbc) {
return
(lbc == UTF8PROC_BOUNDCLASS_START) ? true : // GB1
(lbc == UTF8PROC_BOUNDCLASS_CR && // GB3
tbc == UTF8PROC_BOUNDCLASS_LF) ? false : // ---
(lbc >= UTF8PROC_BOUNDCLASS_CR && lbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true : // GB4
(tbc >= UTF8PROC_BOUNDCLASS_CR && tbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true : // GB5
(lbc == UTF8PROC_BOUNDCLASS_L && // GB6
(tbc == UTF8PROC_BOUNDCLASS_L || // ---
tbc == UTF8PROC_BOUNDCLASS_V || // ---
tbc == UTF8PROC_BOUNDCLASS_LV || // ---
tbc == UTF8PROC_BOUNDCLASS_LVT)) ? false : // ---
((lbc == UTF8PROC_BOUNDCLASS_LV || // GB7
lbc == UTF8PROC_BOUNDCLASS_V) && // ---
(tbc == UTF8PROC_BOUNDCLASS_V || // ---
tbc == UTF8PROC_BOUNDCLASS_T)) ? false : // ---
((lbc == UTF8PROC_BOUNDCLASS_LVT || // GB8
lbc == UTF8PROC_BOUNDCLASS_T) && // ---
tbc == UTF8PROC_BOUNDCLASS_T) ? false : // ---
(tbc == UTF8PROC_BOUNDCLASS_EXTEND || // GB9
tbc == UTF8PROC_BOUNDCLASS_ZWJ || // ---
tbc == UTF8PROC_BOUNDCLASS_SPACINGMARK || // GB9a
lbc == UTF8PROC_BOUNDCLASS_PREPEND) ? false : // GB9b
((lbc == UTF8PROC_BOUNDCLASS_E_BASE || // GB10 (requires additional handling below)
lbc == UTF8PROC_BOUNDCLASS_E_BASE_GAZ) && // ----
tbc == UTF8PROC_BOUNDCLASS_E_MODIFIER) ? false : // ----
(lbc == UTF8PROC_BOUNDCLASS_ZWJ && // GB11
(tbc == UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ || // ----
tbc == UTF8PROC_BOUNDCLASS_E_BASE_GAZ)) ? false : // ----
(lbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR && // GB12/13 (requires additional handling below)
tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR) ? false : // ----
true; // GB999
}
static utf8proc_bool grapheme_break_extended(int lbc, int tbc, utf8proc_int32_t *state)
{
int lbc_override = lbc;
if (state && *state != UTF8PROC_BOUNDCLASS_START)
lbc_override = *state;
utf8proc_bool break_permitted = grapheme_break_simple(lbc_override, tbc);
if (state) {
// Special support for GB 12/13 made possible by GB999. After two RI
// class codepoints we want to force a break. Do this by resetting the
// second RI's bound class to UTF8PROC_BOUNDCLASS_OTHER, to force a break
// after that character according to GB999 (unless of course such a break is
// forbidden by a different rule such as GB9).
if (*state == tbc && tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR)
*state = UTF8PROC_BOUNDCLASS_OTHER;
// Special support for GB10. Fold any EXTEND codepoints into the previous
// boundclass if we're dealing with an emoji base boundclass.
else if ((*state == UTF8PROC_BOUNDCLASS_E_BASE ||
*state == UTF8PROC_BOUNDCLASS_E_BASE_GAZ) &&
tbc == UTF8PROC_BOUNDCLASS_EXTEND)
*state = UTF8PROC_BOUNDCLASS_E_BASE;
else
*state = tbc;
}
return break_permitted;
}
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
utf8proc_int32_t c1, utf8proc_int32_t c2, utf8proc_int32_t *state) {
return grapheme_break_extended(utf8proc_get_property(c1)->boundclass,
utf8proc_get_property(c2)->boundclass,
state);
}
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
utf8proc_int32_t c1, utf8proc_int32_t c2) {
return utf8proc_grapheme_break_stateful(c1, c2, NULL);
}
static utf8proc_int32_t seqindex_decode_entry(const utf8proc_uint16_t **entry)
{
utf8proc_int32_t entry_cp = **entry;
if ((entry_cp & 0xF800) == 0xD800) {
*entry = *entry + 1;
entry_cp = ((entry_cp & 0x03FF) << 10) | (**entry & 0x03FF);
entry_cp += 0x10000;
}
return entry_cp;
}
static utf8proc_int32_t seqindex_decode_index(const utf8proc_uint32_t seqindex)
{
const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex];
return seqindex_decode_entry(&entry);
}
static utf8proc_ssize_t seqindex_write_char_decomposed(utf8proc_uint16_t seqindex, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
utf8proc_ssize_t written = 0;
const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex & 0x1FFF];
int len = seqindex >> 13;
if (len >= 7) {
len = *entry;
entry++;
}
for (; len >= 0; entry++, len--) {
utf8proc_int32_t entry_cp = seqindex_decode_entry(&entry);
written += utf8proc_decompose_char(entry_cp, dst+written,
(bufsize > written) ? (bufsize - written) : 0, options,
last_boundclass);
if (written < 0) return UTF8PROC_ERROR_OVERFLOW;
}
return written;
}
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c)
{
utf8proc_int32_t cl = utf8proc_get_property(c)->lowercase_seqindex;
return cl != UINT16_MAX ? seqindex_decode_index(cl) : c;
}
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c)
{
utf8proc_int32_t cu = utf8proc_get_property(c)->uppercase_seqindex;
return cu != UINT16_MAX ? seqindex_decode_index(cu) : c;
}
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c)
{
utf8proc_int32_t cu = utf8proc_get_property(c)->titlecase_seqindex;
return cu != UINT16_MAX ? seqindex_decode_index(cu) : c;
}
/* return a character width analogous to wcwidth (except portable and
hopefully less buggy than most system wcwidth functions). */
UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t c) {
return utf8proc_get_property(c)->charwidth;
}
UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t c) {
return utf8proc_get_property(c)->category;
}
UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t c) {
static const char s[][3] = {"Cn","Lu","Ll","Lt","Lm","Lo","Mn","Mc","Me","Nd","Nl","No","Pc","Pd","Ps","Pe","Pi","Pf","Po","Sm","Sc","Sk","So","Zs","Zl","Zp","Cc","Cf","Cs","Co"};
return s[utf8proc_category(c)];
}
#define utf8proc_decompose_lump(replacement_uc) \
return utf8proc_decompose_char((replacement_uc), dst, bufsize, \
options & ~UTF8PROC_LUMP, last_boundclass)
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
const utf8proc_property_t *property;
utf8proc_propval_t category;
utf8proc_int32_t hangul_sindex;
if (uc < 0 || uc >= 0x110000) return UTF8PROC_ERROR_NOTASSIGNED;
property = unsafe_get_property(uc);
category = property->category;
hangul_sindex = uc - UTF8PROC_HANGUL_SBASE;
if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT) {
utf8proc_int32_t hangul_tindex;
if (bufsize >= 1) {
dst[0] = UTF8PROC_HANGUL_LBASE +
hangul_sindex / UTF8PROC_HANGUL_NCOUNT;
if (bufsize >= 2) dst[1] = UTF8PROC_HANGUL_VBASE +
(hangul_sindex % UTF8PROC_HANGUL_NCOUNT) / UTF8PROC_HANGUL_TCOUNT;
}
hangul_tindex = hangul_sindex % UTF8PROC_HANGUL_TCOUNT;
if (!hangul_tindex) return 2;
if (bufsize >= 3) dst[2] = UTF8PROC_HANGUL_TBASE + hangul_tindex;
return 3;
}
}
if (options & UTF8PROC_REJECTNA) {
if (!category) return UTF8PROC_ERROR_NOTASSIGNED;
}
if (options & UTF8PROC_IGNORE) {
if (property->ignorable) return 0;
}
if (options & UTF8PROC_LUMP) {
if (category == UTF8PROC_CATEGORY_ZS) utf8proc_decompose_lump(0x0020);
if (uc == 0x2018 || uc == 0x2019 || uc == 0x02BC || uc == 0x02C8)
utf8proc_decompose_lump(0x0027);
if (category == UTF8PROC_CATEGORY_PD || uc == 0x2212)
utf8proc_decompose_lump(0x002D);
if (uc == 0x2044 || uc == 0x2215) utf8proc_decompose_lump(0x002F);
if (uc == 0x2236) utf8proc_decompose_lump(0x003A);
if (uc == 0x2039 || uc == 0x2329 || uc == 0x3008)
utf8proc_decompose_lump(0x003C);
if (uc == 0x203A || uc == 0x232A || uc == 0x3009)
utf8proc_decompose_lump(0x003E);
if (uc == 0x2216) utf8proc_decompose_lump(0x005C);
if (uc == 0x02C4 || uc == 0x02C6 || uc == 0x2038 || uc == 0x2303)
utf8proc_decompose_lump(0x005E);
if (category == UTF8PROC_CATEGORY_PC || uc == 0x02CD)
utf8proc_decompose_lump(0x005F);
if (uc == 0x02CB) utf8proc_decompose_lump(0x0060);
if (uc == 0x2223) utf8proc_decompose_lump(0x007C);
if (uc == 0x223C) utf8proc_decompose_lump(0x007E);
if ((options & UTF8PROC_NLF2LS) && (options & UTF8PROC_NLF2PS)) {
if (category == UTF8PROC_CATEGORY_ZL ||
category == UTF8PROC_CATEGORY_ZP)
utf8proc_decompose_lump(0x000A);
}
}
if (options & UTF8PROC_STRIPMARK) {
if (category == UTF8PROC_CATEGORY_MN ||
category == UTF8PROC_CATEGORY_MC ||
category == UTF8PROC_CATEGORY_ME) return 0;
}
if (options & UTF8PROC_CASEFOLD) {
if (property->casefold_seqindex != UINT16_MAX) {
return seqindex_write_char_decomposed(property->casefold_seqindex, dst, bufsize, options, last_boundclass);
}
}
if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
if (property->decomp_seqindex != UINT16_MAX &&
(!property->decomp_type || (options & UTF8PROC_COMPAT))) {
return seqindex_write_char_decomposed(property->decomp_seqindex, dst, bufsize, options, last_boundclass);
}
}
if (options & UTF8PROC_CHARBOUND) {
utf8proc_bool boundary;
int tbc = property->boundclass;
boundary = grapheme_break_extended(*last_boundclass, tbc, last_boundclass);
if (boundary) {
if (bufsize >= 1) dst[0] = 0xFFFF;
if (bufsize >= 2) dst[1] = uc;
return 2;
}
}
if (bufsize >= 1) *dst = uc;
return 1;
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
) {
return utf8proc_decompose_custom(str, strlen, buffer, bufsize, options, NULL, NULL);
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options,
utf8proc_custom_func custom_func, void *custom_data
) {
/* strlen will be ignored, if UTF8PROC_NULLTERM is set in options */
utf8proc_ssize_t wpos = 0;
if ((options & UTF8PROC_COMPOSE) && (options & UTF8PROC_DECOMPOSE))
return UTF8PROC_ERROR_INVALIDOPTS;
if ((options & UTF8PROC_STRIPMARK) &&
!(options & UTF8PROC_COMPOSE) && !(options & UTF8PROC_DECOMPOSE))
return UTF8PROC_ERROR_INVALIDOPTS;
{
utf8proc_int32_t uc;
utf8proc_ssize_t rpos = 0;
utf8proc_ssize_t decomp_result;
int boundclass = UTF8PROC_BOUNDCLASS_START;
while (1) {
if (options & UTF8PROC_NULLTERM) {
rpos += utf8proc_iterate(str + rpos, -1, &uc);
/* checking of return value is not necessary,
as 'uc' is < 0 in case of error */
if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
if (rpos < 0) return UTF8PROC_ERROR_OVERFLOW;
if (uc == 0) break;
} else {
if (rpos >= strlen) break;
rpos += utf8proc_iterate(str + rpos, strlen - rpos, &uc);
if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
}
if (custom_func != NULL) {
uc = custom_func(uc, custom_data); /* user-specified custom mapping */
}
decomp_result = utf8proc_decompose_char(
uc, buffer + wpos, (bufsize > wpos) ? (bufsize - wpos) : 0, options,
&boundclass
);
if (decomp_result < 0) return decomp_result;
wpos += decomp_result;
/* prohibiting integer overflows due to too long strings: */
if (wpos < 0 ||
wpos > (utf8proc_ssize_t)(SSIZE_MAX/sizeof(utf8proc_int32_t)/2))
return UTF8PROC_ERROR_OVERFLOW;
}
}
if ((options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && bufsize >= wpos) {
utf8proc_ssize_t pos = 0;
while (pos < wpos-1) {
utf8proc_int32_t uc1, uc2;
const utf8proc_property_t *property1, *property2;
uc1 = buffer[pos];
uc2 = buffer[pos+1];
property1 = unsafe_get_property(uc1);
property2 = unsafe_get_property(uc2);
if (property1->combining_class > property2->combining_class &&
property2->combining_class > 0) {
buffer[pos] = uc2;
buffer[pos+1] = uc1;
if (pos > 0) pos--; else pos++;
} else {
pos++;
}
}
}
return wpos;
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
/* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored */
if (options & (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS | UTF8PROC_STRIPCC)) {
utf8proc_ssize_t rpos;
utf8proc_ssize_t wpos = 0;
utf8proc_int32_t uc;
for (rpos = 0; rpos < length; rpos++) {
uc = buffer[rpos];
if (uc == 0x000D && rpos < length-1 && buffer[rpos+1] == 0x000A) rpos++;
if (uc == 0x000A || uc == 0x000D || uc == 0x0085 ||
((options & UTF8PROC_STRIPCC) && (uc == 0x000B || uc == 0x000C))) {
if (options & UTF8PROC_NLF2LS) {
if (options & UTF8PROC_NLF2PS) {
buffer[wpos++] = 0x000A;
} else {
buffer[wpos++] = 0x2028;
}
} else {
if (options & UTF8PROC_NLF2PS) {
buffer[wpos++] = 0x2029;
} else {
buffer[wpos++] = 0x0020;
}
}
} else if ((options & UTF8PROC_STRIPCC) &&
(uc < 0x0020 || (uc >= 0x007F && uc < 0x00A0))) {
if (uc == 0x0009) buffer[wpos++] = 0x0020;
} else {
buffer[wpos++] = uc;
}
}
length = wpos;
}
if (options & UTF8PROC_COMPOSE) {
utf8proc_int32_t *starter = NULL;
utf8proc_int32_t current_char;
const utf8proc_property_t *starter_property = NULL, *current_property;
utf8proc_propval_t max_combining_class = -1;
utf8proc_ssize_t rpos;
utf8proc_ssize_t wpos = 0;
utf8proc_int32_t composition;
for (rpos = 0; rpos < length; rpos++) {
current_char = buffer[rpos];
current_property = unsafe_get_property(current_char);
if (starter && current_property->combining_class > max_combining_class) {
/* combination perhaps possible */
utf8proc_int32_t hangul_lindex;
utf8proc_int32_t hangul_sindex;
hangul_lindex = *starter - UTF8PROC_HANGUL_LBASE;
if (hangul_lindex >= 0 && hangul_lindex < UTF8PROC_HANGUL_LCOUNT) {
utf8proc_int32_t hangul_vindex;
hangul_vindex = current_char - UTF8PROC_HANGUL_VBASE;
if (hangul_vindex >= 0 && hangul_vindex < UTF8PROC_HANGUL_VCOUNT) {
*starter = UTF8PROC_HANGUL_SBASE +
(hangul_lindex * UTF8PROC_HANGUL_VCOUNT + hangul_vindex) *
UTF8PROC_HANGUL_TCOUNT;
starter_property = NULL;
continue;
}
}
hangul_sindex = *starter - UTF8PROC_HANGUL_SBASE;
if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT &&
(hangul_sindex % UTF8PROC_HANGUL_TCOUNT) == 0) {
utf8proc_int32_t hangul_tindex;
hangul_tindex = current_char - UTF8PROC_HANGUL_TBASE;
if (hangul_tindex >= 0 && hangul_tindex < UTF8PROC_HANGUL_TCOUNT) {
*starter += hangul_tindex;
starter_property = NULL;
continue;
}
}
if (!starter_property) {
starter_property = unsafe_get_property(*starter);
}
if (starter_property->comb_index < 0x8000 &&
current_property->comb_index != UINT16_MAX &&
current_property->comb_index >= 0x8000) {
int sidx = starter_property->comb_index;
int idx = (current_property->comb_index & 0x3FFF) - utf8proc_combinations[sidx];
if (idx >= 0 && idx <= utf8proc_combinations[sidx + 1] ) {
idx += sidx + 2;
if (current_property->comb_index & 0x4000) {
composition = (utf8proc_combinations[idx] << 16) | utf8proc_combinations[idx+1];
} else
composition = utf8proc_combinations[idx];
if (composition > 0 && (!(options & UTF8PROC_STABLE) ||
!(unsafe_get_property(composition)->comp_exclusion))) {
*starter = composition;
starter_property = NULL;
continue;
}
}
}
}
buffer[wpos] = current_char;
if (current_property->combining_class) {
if (current_property->combining_class > max_combining_class) {
max_combining_class = current_property->combining_class;
}
} else {
starter = buffer + wpos;
starter_property = NULL;
max_combining_class = -1;
}
wpos++;
}
length = wpos;
}
return length;
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
/* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored
ASSERT: 'buffer' has one spare byte of free space at the end! */
length = utf8proc_normalize_utf32(buffer, length, options);
if (length < 0) return length;
{
utf8proc_ssize_t rpos, wpos = 0;
utf8proc_int32_t uc;
if (options & UTF8PROC_CHARBOUND) {
for (rpos = 0; rpos < length; rpos++) {
uc = buffer[rpos];
wpos += unsafe_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
}
} else {
for (rpos = 0; rpos < length; rpos++) {
uc = buffer[rpos];
wpos += utf8proc_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
}
}
((utf8proc_uint8_t *)buffer)[wpos] = 0;
return wpos;
}
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
) {
return utf8proc_map_custom(str, strlen, dstptr, options, NULL, NULL);
}
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options,
utf8proc_custom_func custom_func, void *custom_data
) {
utf8proc_int32_t *buffer;
utf8proc_ssize_t result;
*dstptr = NULL;
result = utf8proc_decompose_custom(str, strlen, NULL, 0, options, custom_func, custom_data);
if (result < 0) return result;
buffer = (utf8proc_int32_t *) malloc(result * sizeof(utf8proc_int32_t) + 1);
if (!buffer) return UTF8PROC_ERROR_NOMEM;
result = utf8proc_decompose_custom(str, strlen, buffer, result, options, custom_func, custom_data);
if (result < 0) {
free(buffer);
return result;
}
result = utf8proc_reencode(buffer, result, options);
if (result < 0) {
free(buffer);
return result;
}
{
utf8proc_int32_t *newptr;
newptr = (utf8proc_int32_t *) realloc(buffer, (size_t)result+1);
if (newptr) buffer = newptr;
}
*dstptr = (utf8proc_uint8_t *)buffer;
return result;
}
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_DECOMPOSE);
return retval;
}
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_COMPOSE);
return retval;
}
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
return retval;
}
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str) {
utf8proc_uint8_t *retval;
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
return retval;
}

689
lib/utf8proc-2a2f97e1/utf8proc.h Normal file
View File

@ -0,0 +1,689 @@
/*
* Copyright (c) 2015 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
* Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
/**
* @mainpage
*
* utf8proc is a free/open-source (MIT/expat licensed) C library
* providing Unicode normalization, case-folding, and other operations
* for strings in the UTF-8 encoding, supporting Unicode version
* 9.0.0. See the utf8proc home page (http://julialang.org/utf8proc/)
* for downloads and other information, or the source code on github
* (https://github.com/JuliaLang/utf8proc).
*
* For the utf8proc API documentation, see: @ref utf8proc.h
*
* The features of utf8proc include:
*
* - Transformation of strings (@ref utf8proc_map) to:
* - decompose (@ref UTF8PROC_DECOMPOSE) or compose (@ref UTF8PROC_COMPOSE) Unicode combining characters (http://en.wikipedia.org/wiki/Combining_character)
* - canonicalize Unicode compatibility characters (@ref UTF8PROC_COMPAT)
* - strip "ignorable" (@ref UTF8PROC_IGNORE) characters, control characters (@ref UTF8PROC_STRIPCC), or combining characters such as accents (@ref UTF8PROC_STRIPMARK)
* - case-folding (@ref UTF8PROC_CASEFOLD)
* - Unicode normalization: @ref utf8proc_NFD, @ref utf8proc_NFC, @ref utf8proc_NFKD, @ref utf8proc_NFKC
* - Detecting grapheme boundaries (@ref utf8proc_grapheme_break and @ref UTF8PROC_CHARBOUND)
* - Character-width computation: @ref utf8proc_charwidth
* - Classification of characters by Unicode category: @ref utf8proc_category and @ref utf8proc_category_string
* - Encode (@ref utf8proc_encode_char) and decode (@ref utf8proc_iterate) Unicode codepoints to/from UTF-8.
*/
/** @file */
#ifndef UTF8PROC_H
#define UTF8PROC_H
/** @name API version
*
* The utf8proc API version MAJOR.MINOR.PATCH, following
* semantic-versioning rules (http://semver.org) based on API
* compatibility.
*
* This is also returned at runtime by @ref utf8proc_version; however, the
* runtime version may append a string like "-dev" to the version number
* for prerelease versions.
*
* @note The shared-library version number in the Makefile
* (and CMakeLists.txt, and MANIFEST) may be different,
* being based on ABI compatibility rather than API compatibility.
*/
/** @{ */
/** The MAJOR version number (increased when backwards API compatibility is broken). */
#define UTF8PROC_VERSION_MAJOR 2
/** The MINOR version number (increased when new functionality is added in a backwards-compatible manner). */
#define UTF8PROC_VERSION_MINOR 1
/** The PATCH version (increased for fixes that do not change the API). */
#define UTF8PROC_VERSION_PATCH 0
/** @} */
#include <stdlib.h>
#if defined(_MSC_VER) && _MSC_VER < 1800
// MSVC prior to 2013 lacked stdbool.h and inttypes.h
typedef signed char utf8proc_int8_t;
typedef unsigned char utf8proc_uint8_t;
typedef short utf8proc_int16_t;
typedef unsigned short utf8proc_uint16_t;
typedef int utf8proc_int32_t;
typedef unsigned int utf8proc_uint32_t;
# ifdef _WIN64
typedef __int64 utf8proc_ssize_t;
typedef unsigned __int64 utf8proc_size_t;
# else
typedef int utf8proc_ssize_t;
typedef unsigned int utf8proc_size_t;
# endif
# ifndef __cplusplus
// emulate C99 bool
typedef unsigned char utf8proc_bool;
# ifndef __bool_true_false_are_defined
# define false 0
# define true 1
# define __bool_true_false_are_defined 1
# endif
# else
typedef bool utf8proc_bool;
# endif
#else
# include <stddef.h>
# include <stdbool.h>
# include <inttypes.h>
typedef int8_t utf8proc_int8_t;
typedef uint8_t utf8proc_uint8_t;
typedef int16_t utf8proc_int16_t;
typedef uint16_t utf8proc_uint16_t;
typedef int32_t utf8proc_int32_t;
typedef uint32_t utf8proc_uint32_t;
typedef size_t utf8proc_size_t;
typedef ptrdiff_t utf8proc_ssize_t;
typedef bool utf8proc_bool;
#endif
#include <limits.h>
#define UTF8PROC_DLLEXPORT
#ifdef __cplusplus
extern "C" {
#endif
#ifndef SSIZE_MAX
#define SSIZE_MAX ((size_t)SIZE_MAX/2)
#endif
#ifndef UINT16_MAX
# define UINT16_MAX 65535U
#endif
/**
* Option flags used by several functions in the library.
*/
typedef enum {
/** The given UTF-8 input is NULL terminated. */
UTF8PROC_NULLTERM = (1<<0),
/** Unicode Versioning Stability has to be respected. */
UTF8PROC_STABLE = (1<<1),
/** Compatibility decomposition (i.e. formatting information is lost). */
UTF8PROC_COMPAT = (1<<2),
/** Return a result with decomposed characters. */
UTF8PROC_COMPOSE = (1<<3),
/** Return a result with decomposed characters. */
UTF8PROC_DECOMPOSE = (1<<4),
/** Strip "default ignorable characters" such as SOFT-HYPHEN or ZERO-WIDTH-SPACE. */
UTF8PROC_IGNORE = (1<<5),
/** Return an error, if the input contains unassigned codepoints. */
UTF8PROC_REJECTNA = (1<<6),
/**
* Indicating that NLF-sequences (LF, CRLF, CR, NEL) are representing a
* line break, and should be converted to the codepoint for line
* separation (LS).
*/
UTF8PROC_NLF2LS = (1<<7),
/**
* Indicating that NLF-sequences are representing a paragraph break, and
* should be converted to the codepoint for paragraph separation
* (PS).
*/
UTF8PROC_NLF2PS = (1<<8),
/** Indicating that the meaning of NLF-sequences is unknown. */
UTF8PROC_NLF2LF = (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS),
/** Strips and/or convers control characters.
*
* NLF-sequences are transformed into space, except if one of the
* NLF2LS/PS/LF options is given. HorizontalTab (HT) and FormFeed (FF)
* are treated as a NLF-sequence in this case. All other control
* characters are simply removed.
*/
UTF8PROC_STRIPCC = (1<<9),
/**
* Performs unicode case folding, to be able to do a case-insensitive
* string comparison.
*/
UTF8PROC_CASEFOLD = (1<<10),
/**
* Inserts 0xFF bytes at the beginning of each sequence which is
* representing a single grapheme cluster (see UAX#29).
*/
UTF8PROC_CHARBOUND = (1<<11),
/** Lumps certain characters together.
*
* E.g. HYPHEN U+2010 and MINUS U+2212 to ASCII "-". See lump.md for details.
*
* If NLF2LF is set, this includes a transformation of paragraph and
* line separators to ASCII line-feed (LF).
*/
UTF8PROC_LUMP = (1<<12),
/** Strips all character markings.
*
* This includes non-spacing, spacing and enclosing (i.e. accents).
* @note This option works only with @ref UTF8PROC_COMPOSE or
* @ref UTF8PROC_DECOMPOSE
*/
UTF8PROC_STRIPMARK = (1<<13),
} utf8proc_option_t;
/** @name Error codes
* Error codes being returned by almost all functions.
*/
/** @{ */
/** Memory could not be allocated. */
#define UTF8PROC_ERROR_NOMEM -1
/** The given string is too long to be processed. */
#define UTF8PROC_ERROR_OVERFLOW -2
/** The given string is not a legal UTF-8 string. */
#define UTF8PROC_ERROR_INVALIDUTF8 -3
/** The @ref UTF8PROC_REJECTNA flag was set and an unassigned codepoint was found. */
#define UTF8PROC_ERROR_NOTASSIGNED -4
/** Invalid options have been used. */
#define UTF8PROC_ERROR_INVALIDOPTS -5
/** @} */
/* @name Types */
/** Holds the value of a property. */
typedef utf8proc_int16_t utf8proc_propval_t;
/** Struct containing information about a codepoint. */
typedef struct utf8proc_property_struct {
/**
* Unicode category.
* @see utf8proc_category_t.
*/
utf8proc_propval_t category;
utf8proc_propval_t combining_class;
/**
* Bidirectional class.
* @see utf8proc_bidi_class_t.
*/
utf8proc_propval_t bidi_class;
/**
* @anchor Decomposition type.
* @see utf8proc_decomp_type_t.
*/
utf8proc_propval_t decomp_type;
utf8proc_uint16_t decomp_seqindex;
utf8proc_uint16_t casefold_seqindex;
utf8proc_uint16_t uppercase_seqindex;
utf8proc_uint16_t lowercase_seqindex;
utf8proc_uint16_t titlecase_seqindex;
utf8proc_uint16_t comb_index;
unsigned bidi_mirrored:1;
unsigned comp_exclusion:1;
/**
* Can this codepoint be ignored?
*
* Used by @ref utf8proc_decompose_char when @ref UTF8PROC_IGNORE is
* passed as an option.
*/
unsigned ignorable:1;
unsigned control_boundary:1;
/** The width of the codepoint. */
unsigned charwidth:2;
unsigned pad:2;
/**
* Boundclass.
* @see utf8proc_boundclass_t.
*/
unsigned boundclass:8;
} utf8proc_property_t;
/** Unicode categories. */
typedef enum {
UTF8PROC_CATEGORY_CN = 0, /**< Other, not assigned */
UTF8PROC_CATEGORY_LU = 1, /**< Letter, uppercase */
UTF8PROC_CATEGORY_LL = 2, /**< Letter, lowercase */
UTF8PROC_CATEGORY_LT = 3, /**< Letter, titlecase */
UTF8PROC_CATEGORY_LM = 4, /**< Letter, modifier */
UTF8PROC_CATEGORY_LO = 5, /**< Letter, other */
UTF8PROC_CATEGORY_MN = 6, /**< Mark, nonspacing */
UTF8PROC_CATEGORY_MC = 7, /**< Mark, spacing combining */
UTF8PROC_CATEGORY_ME = 8, /**< Mark, enclosing */
UTF8PROC_CATEGORY_ND = 9, /**< Number, decimal digit */
UTF8PROC_CATEGORY_NL = 10, /**< Number, letter */
UTF8PROC_CATEGORY_NO = 11, /**< Number, other */
UTF8PROC_CATEGORY_PC = 12, /**< Punctuation, connector */
UTF8PROC_CATEGORY_PD = 13, /**< Punctuation, dash */
UTF8PROC_CATEGORY_PS = 14, /**< Punctuation, open */
UTF8PROC_CATEGORY_PE = 15, /**< Punctuation, close */
UTF8PROC_CATEGORY_PI = 16, /**< Punctuation, initial quote */
UTF8PROC_CATEGORY_PF = 17, /**< Punctuation, final quote */
UTF8PROC_CATEGORY_PO = 18, /**< Punctuation, other */
UTF8PROC_CATEGORY_SM = 19, /**< Symbol, math */
UTF8PROC_CATEGORY_SC = 20, /**< Symbol, currency */
UTF8PROC_CATEGORY_SK = 21, /**< Symbol, modifier */
UTF8PROC_CATEGORY_SO = 22, /**< Symbol, other */
UTF8PROC_CATEGORY_ZS = 23, /**< Separator, space */
UTF8PROC_CATEGORY_ZL = 24, /**< Separator, line */
UTF8PROC_CATEGORY_ZP = 25, /**< Separator, paragraph */
UTF8PROC_CATEGORY_CC = 26, /**< Other, control */
UTF8PROC_CATEGORY_CF = 27, /**< Other, format */
UTF8PROC_CATEGORY_CS = 28, /**< Other, surrogate */
UTF8PROC_CATEGORY_CO = 29, /**< Other, private use */
} utf8proc_category_t;
/** Bidirectional character classes. */
typedef enum {
UTF8PROC_BIDI_CLASS_L = 1, /**< Left-to-Right */
UTF8PROC_BIDI_CLASS_LRE = 2, /**< Left-to-Right Embedding */
UTF8PROC_BIDI_CLASS_LRO = 3, /**< Left-to-Right Override */
UTF8PROC_BIDI_CLASS_R = 4, /**< Right-to-Left */
UTF8PROC_BIDI_CLASS_AL = 5, /**< Right-to-Left Arabic */
UTF8PROC_BIDI_CLASS_RLE = 6, /**< Right-to-Left Embedding */
UTF8PROC_BIDI_CLASS_RLO = 7, /**< Right-to-Left Override */
UTF8PROC_BIDI_CLASS_PDF = 8, /**< Pop Directional Format */
UTF8PROC_BIDI_CLASS_EN = 9, /**< European Number */
UTF8PROC_BIDI_CLASS_ES = 10, /**< European Separator */
UTF8PROC_BIDI_CLASS_ET = 11, /**< European Number Terminator */
UTF8PROC_BIDI_CLASS_AN = 12, /**< Arabic Number */
UTF8PROC_BIDI_CLASS_CS = 13, /**< Common Number Separator */
UTF8PROC_BIDI_CLASS_NSM = 14, /**< Nonspacing Mark */
UTF8PROC_BIDI_CLASS_BN = 15, /**< Boundary Neutral */
UTF8PROC_BIDI_CLASS_B = 16, /**< Paragraph Separator */
UTF8PROC_BIDI_CLASS_S = 17, /**< Segment Separator */
UTF8PROC_BIDI_CLASS_WS = 18, /**< Whitespace */
UTF8PROC_BIDI_CLASS_ON = 19, /**< Other Neutrals */
UTF8PROC_BIDI_CLASS_LRI = 20, /**< Left-to-Right Isolate */
UTF8PROC_BIDI_CLASS_RLI = 21, /**< Right-to-Left Isolate */
UTF8PROC_BIDI_CLASS_FSI = 22, /**< First Strong Isolate */
UTF8PROC_BIDI_CLASS_PDI = 23, /**< Pop Directional Isolate */
} utf8proc_bidi_class_t;
/** Decomposition type. */
typedef enum {
UTF8PROC_DECOMP_TYPE_FONT = 1, /**< Font */
UTF8PROC_DECOMP_TYPE_NOBREAK = 2, /**< Nobreak */
UTF8PROC_DECOMP_TYPE_INITIAL = 3, /**< Initial */
UTF8PROC_DECOMP_TYPE_MEDIAL = 4, /**< Medial */
UTF8PROC_DECOMP_TYPE_FINAL = 5, /**< Final */
UTF8PROC_DECOMP_TYPE_ISOLATED = 6, /**< Isolated */
UTF8PROC_DECOMP_TYPE_CIRCLE = 7, /**< Circle */
UTF8PROC_DECOMP_TYPE_SUPER = 8, /**< Super */
UTF8PROC_DECOMP_TYPE_SUB = 9, /**< Sub */
UTF8PROC_DECOMP_TYPE_VERTICAL = 10, /**< Vertical */
UTF8PROC_DECOMP_TYPE_WIDE = 11, /**< Wide */
UTF8PROC_DECOMP_TYPE_NARROW = 12, /**< Narrow */
UTF8PROC_DECOMP_TYPE_SMALL = 13, /**< Small */
UTF8PROC_DECOMP_TYPE_SQUARE = 14, /**< Square */
UTF8PROC_DECOMP_TYPE_FRACTION = 15, /**< Fraction */
UTF8PROC_DECOMP_TYPE_COMPAT = 16, /**< Compat */
} utf8proc_decomp_type_t;
/** Boundclass property. (TR29) */
typedef enum {
UTF8PROC_BOUNDCLASS_START = 0, /**< Start */
UTF8PROC_BOUNDCLASS_OTHER = 1, /**< Other */
UTF8PROC_BOUNDCLASS_CR = 2, /**< Cr */
UTF8PROC_BOUNDCLASS_LF = 3, /**< Lf */
UTF8PROC_BOUNDCLASS_CONTROL = 4, /**< Control */
UTF8PROC_BOUNDCLASS_EXTEND = 5, /**< Extend */
UTF8PROC_BOUNDCLASS_L = 6, /**< L */
UTF8PROC_BOUNDCLASS_V = 7, /**< V */
UTF8PROC_BOUNDCLASS_T = 8, /**< T */
UTF8PROC_BOUNDCLASS_LV = 9, /**< Lv */
UTF8PROC_BOUNDCLASS_LVT = 10, /**< Lvt */
UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR = 11, /**< Regional indicator */
UTF8PROC_BOUNDCLASS_SPACINGMARK = 12, /**< Spacingmark */
UTF8PROC_BOUNDCLASS_PREPEND = 13, /**< Prepend */
UTF8PROC_BOUNDCLASS_ZWJ = 14, /**< Zero Width Joiner */
UTF8PROC_BOUNDCLASS_E_BASE = 15, /**< Emoji Base */
UTF8PROC_BOUNDCLASS_E_MODIFIER = 16, /**< Emoji Modifier */
UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ = 17, /**< Glue_After_ZWJ */
UTF8PROC_BOUNDCLASS_E_BASE_GAZ = 18, /**< E_BASE + GLUE_AFTER_ZJW */
} utf8proc_boundclass_t;
/**
* Function pointer type passed to @ref utf8proc_map_custom and
* @ref utf8proc_decompose_custom, which is used to specify a user-defined
* mapping of codepoints to be applied in conjunction with other mappings.
*/
typedef utf8proc_int32_t (*utf8proc_custom_func)(utf8proc_int32_t codepoint, void *data);
/**
* Array containing the byte lengths of a UTF-8 encoded codepoint based
* on the first byte.
*/
UTF8PROC_DLLEXPORT extern const utf8proc_int8_t utf8proc_utf8class[256];
/**
* Returns the utf8proc API version as a string MAJOR.MINOR.PATCH
* (http://semver.org format), possibly with a "-dev" suffix for
* development versions.
*/
UTF8PROC_DLLEXPORT const char *utf8proc_version(void);
/**
* Returns an informative error string for the given utf8proc error code
* (e.g. the error codes returned by @ref utf8proc_map).
*/
UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode);
/**
* Reads a single codepoint from the UTF-8 sequence being pointed to by `str`.
* The maximum number of bytes read is `strlen`, unless `strlen` is
* negative (in which case up to 4 bytes are read).
*
* If a valid codepoint could be read, it is stored in the variable
* pointed to by `codepoint_ref`, otherwise that variable will be set to -1.
* In case of success, the number of bytes read is returned; otherwise, a
* negative error code is returned.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *codepoint_ref);
/**
* Check if a codepoint is valid (regardless of whether it has been
* assigned a value by the current Unicode standard).
*
* @return 1 if the given `codepoint` is valid and otherwise return 0.
*/
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t codepoint);
/**
* Encodes the codepoint as an UTF-8 string in the byte array pointed
* to by `dst`. This array must be at least 4 bytes long.
*
* In case of success the number of bytes written is returned, and
* otherwise 0 is returned.
*
* This function does not check whether `codepoint` is valid Unicode.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t codepoint, utf8proc_uint8_t *dst);
/**
* Look up the properties for a given codepoint.
*
* @param codepoint The Unicode codepoint.
*
* @returns
* A pointer to a (constant) struct containing information about
* the codepoint.
* @par
* If the codepoint is unassigned or invalid, a pointer to a special struct is
* returned in which `category` is 0 (@ref UTF8PROC_CATEGORY_CN).
*/
UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t codepoint);
/** Decompose a codepoint into an array of codepoints.
*
* @param codepoint the codepoint.
* @param dst the destination buffer.
* @param bufsize the size of the destination buffer.
* @param options one or more of the following flags:
* - @ref UTF8PROC_REJECTNA - return an error `codepoint` is unassigned
* - @ref UTF8PROC_IGNORE - strip "default ignorable" codepoints
* - @ref UTF8PROC_CASEFOLD - apply Unicode casefolding
* - @ref UTF8PROC_COMPAT - replace certain codepoints with their
* compatibility decomposition
* - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
* - @ref UTF8PROC_LUMP - lump certain different codepoints together
* - @ref UTF8PROC_STRIPMARK - remove all character marks
* @param last_boundclass
* Pointer to an integer variable containing
* the previous codepoint's boundary class if the @ref UTF8PROC_CHARBOUND
* option is used. Otherwise, this parameter is ignored.
*
* @return
* In case of success, the number of codepoints written is returned; in case
* of an error, a negative error code is returned (@ref utf8proc_errmsg).
* @par
* If the number of written codepoints would be bigger than `bufsize`, the
* required buffer size is returned, while the buffer will be overwritten with
* undefined data.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(
utf8proc_int32_t codepoint, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize,
utf8proc_option_t options, int *last_boundclass
);
/**
* The same as @ref utf8proc_decompose_char, but acts on a whole UTF-8
* string and orders the decomposed sequences correctly.
*
* If the @ref UTF8PROC_NULLTERM flag in `options` is set, processing
* will be stopped, when a NULL byte is encounted, otherwise `strlen`
* bytes are processed. The result (in the form of 32-bit unicode
* codepoints) is written into the buffer being pointed to by
* `buffer` (which must contain at least `bufsize` entries). In case of
* success, the number of codepoints written is returned; in case of an
* error, a negative error code is returned (@ref utf8proc_errmsg).
* See @ref utf8proc_decompose_custom to supply additional transformations.
*
* If the number of written codepoints would be bigger than `bufsize`, the
* required buffer size is returned, while the buffer will be overwritten with
* undefined data.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
);
/**
* The same as @ref utf8proc_decompose, but also takes a `custom_func` mapping function
* that is called on each codepoint in `str` before any other transformations
* (along with a `custom_data` pointer that is passed through to `custom_func`).
* The `custom_func` argument is ignored if it is `NULL`. See also @ref utf8proc_map_custom.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options,
utf8proc_custom_func custom_func, void *custom_data
);
/**
* Normalizes the sequence of `length` codepoints pointed to by `buffer`
* in-place (i.e., the result is also stored in `buffer`).
*
* @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
* @param length the length (in codepoints) of the buffer.
* @param options a bitwise or (`|`) of one or more of the following flags:
* - @ref UTF8PROC_NLF2LS - convert LF, CRLF, CR and NEL into LS
* - @ref UTF8PROC_NLF2PS - convert LF, CRLF, CR and NEL into PS
* - @ref UTF8PROC_NLF2LF - convert LF, CRLF, CR and NEL into LF
* - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
* - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
* codepoints
* - @ref UTF8PROC_STABLE - prohibit combining characters that would violate
* the unicode versioning stability
*
* @return
* In case of success, the length (in codepoints) of the normalized UTF-32 string is
* returned; otherwise, a negative error code is returned (@ref utf8proc_errmsg).
*
* @warning The entries of the array pointed to by `str` have to be in the
* range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
/**
* Reencodes the sequence of `length` codepoints pointed to by `buffer`
* UTF-8 data in-place (i.e., the result is also stored in `buffer`).
* Can optionally normalize the UTF-32 sequence prior to UTF-8 conversion.
*
* @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
* @param length the length (in codepoints) of the buffer.
* @param options a bitwise or (`|`) of one or more of the following flags:
* - @ref UTF8PROC_NLF2LS - convert LF, CRLF, CR and NEL into LS
* - @ref UTF8PROC_NLF2PS - convert LF, CRLF, CR and NEL into PS
* - @ref UTF8PROC_NLF2LF - convert LF, CRLF, CR and NEL into LF
* - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
* - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
* codepoints
* - @ref UTF8PROC_STABLE - prohibit combining characters that would violate
* the unicode versioning stability
* - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
*
* @return
* In case of success, the length (in bytes) of the resulting nul-terminated
* UTF-8 string is returned; otherwise, a negative error code is returned
* (@ref utf8proc_errmsg).
*
* @warning The amount of free space pointed to by `buffer` must
* exceed the amount of the input data by one byte, and the
* entries of the array pointed to by `str` have to be in the
* range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
/**
* Given a pair of consecutive codepoints, return whether a grapheme break is
* permitted between them (as defined by the extended grapheme clusters in UAX#29).
*
* @param state Beginning with Version 29 (Unicode 9.0.0), this algorithm requires
* state to break graphemes. This state can be passed in as a pointer
* in the `state` argument and should initially be set to 0. If the
* state is not passed in (i.e. a null pointer is passed), UAX#29 rules
* GB10/12/13 which require this state will not be applied, essentially
* matching the rules in Unicode 8.0.0.
*
* @warning If the state parameter is used, `utf8proc_grapheme_break_stateful` must
* be called IN ORDER on ALL potential breaks in a string.
*/
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2, utf8proc_int32_t *state);
/**
* Same as @ref utf8proc_grapheme_break_stateful, except without support for the
* Unicode 9 additions to the algorithm. Supported for legacy reasons.
*/
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2);
/**
* Given a codepoint `c`, return the codepoint of the corresponding
* lower-case character, if any; otherwise (if there is no lower-case
* variant, or if `c` is not a valid codepoint) return `c`.
*/
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c);
/**
* Given a codepoint `c`, return the codepoint of the corresponding
* upper-case character, if any; otherwise (if there is no upper-case
* variant, or if `c` is not a valid codepoint) return `c`.
*/
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c);
/**
* Given a codepoint `c`, return the codepoint of the corresponding
* title-case character, if any; otherwise (if there is no title-case
* variant, or if `c` is not a valid codepoint) return `c`.
*/
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c);
/**
* Given a codepoint, return a character width analogous to `wcwidth(codepoint)`,
* except that a width of 0 is returned for non-printable codepoints
* instead of -1 as in `wcwidth`.
*
* @note
* If you want to check for particular types of non-printable characters,
* (analogous to `isprint` or `iscntrl`), use @ref utf8proc_category. */
UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t codepoint);
/**
* Return the Unicode category for the codepoint (one of the
* @ref utf8proc_category_t constants.)
*/
UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t codepoint);
/**
* Return the two-letter (nul-terminated) Unicode category string for
* the codepoint (e.g. `"Lu"` or `"Co"`).
*/
UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t codepoint);
/**
* Maps the given UTF-8 string pointed to by `str` to a new UTF-8
* string, allocated dynamically by `malloc` and returned via `dstptr`.
*
* If the @ref UTF8PROC_NULLTERM flag in the `options` field is set,
* the length is determined by a NULL terminator, otherwise the
* parameter `strlen` is evaluated to determine the string length, but
* in any case the result will be NULL terminated (though it might
* contain NULL characters with the string if `str` contained NULL
* characters). Other flags in the `options` field are passed to the
* functions defined above, and regarded as described. See also
* @ref utfproc_map_custom to supply a custom codepoint transformation.
*
* In case of success the length of the new string is returned,
* otherwise a negative error code is returned.
*
* @note The memory of the new UTF-8 string will have been allocated
* with `malloc`, and should therefore be deallocated with `free`.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
);
/**
* Like @ref utf8proc_map, but also takes a `custom_func` mapping function
* that is called on each codepoint in `str` before any other transformations
* (along with a `custom_data` pointer that is passed through to `custom_func`).
* The `custom_func` argument is ignored if it is `NULL`.
*/
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options,
utf8proc_custom_func custom_func, void *custom_data
);
/** @name Unicode normalization
*
* Returns a pointer to newly allocated memory of a NFD, NFC, NFKD or NFKC
* normalized version of the null-terminated string `str`. These
* are shortcuts to calling @ref utf8proc_map with @ref UTF8PROC_NULLTERM
* combined with @ref UTF8PROC_STABLE and flags indicating the normalization.
*/
/** @{ */
/** NFD normalization (@ref UTF8PROC_DECOMPOSE). */
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str);
/** NFC normalization (@ref UTF8PROC_COMPOSE). */
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str);
/** NFKD normalization (@ref UTF8PROC_DECOMPOSE and @ref UTF8PROC_COMPAT). */
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str);
/** NFKC normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT). */
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str);
/** @} */
#ifdef __cplusplus
}
#endif
#endif

14386
lib/utf8proc-2a2f97e1/utf8proc_data.c Normal file
View File

File diff suppressed because it is too large Load Diff

20
lib/utf8proc-2a2f97e1/utils.cmake Normal file
View File

@ -0,0 +1,20 @@
function (disallow_intree_builds)
# Adapted from LLVM's toplevel CMakeLists.txt file
if( CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR AND NOT MSVC_IDE )
message(FATAL_ERROR "
In-source builds are not allowed. CMake would overwrite the
makefiles distributed with utf8proc. Please create a directory
and run cmake from there. Building in a subdirectory is
fine, e.g.:
mkdir build
cd build
cmake ..
This process created the file `CMakeCache.txt' and the
directory `CMakeFiles'. Please delete them.
")
endif()
endfunction()

23
lib/utf8proc.patch Normal file
View File

@ -0,0 +1,23 @@
diff --git a/lib/utf8proc-2a2f97e1/utf8proc.h b/lib/utf8proc-2a2f97e1/utf8proc.h
index 64155a1..2fca528 100644
--- a/lib/utf8proc-2a2f97e1/utf8proc.h
+++ b/lib/utf8proc-2a2f97e1/utf8proc.h
@@ -120,17 +120,7 @@ typedef bool utf8proc_bool;
#endif
#include <limits.h>
-#ifdef _WIN32
-# ifdef UTF8PROC_EXPORTS
-# define UTF8PROC_DLLEXPORT __declspec(dllexport)
-# else
-# define UTF8PROC_DLLEXPORT __declspec(dllimport)
-# endif
-#elif __GNUC__ >= 4
-# define UTF8PROC_DLLEXPORT __attribute__ ((visibility("default")))
-#else
-# define UTF8PROC_DLLEXPORT
-#endif
+#define UTF8PROC_DLLEXPORT
#ifdef __cplusplus
extern "C" {

13
lib/utfcpp-2.3.5/CMakeLists.txt Normal file
View File

@ -0,0 +1,13 @@
cmake_minimum_required (VERSION 2.8.0)
project (utf8cpp)
include_directories ("${PROJECT_SOURCE_DIR}/source")
add_executable(smoke ${PROJECT_SOURCE_DIR}/test_drivers/smoke_test/test.cpp)
add_executable(negative ${PROJECT_SOURCE_DIR}/test_drivers/negative/negative.cpp)
add_executable(utf8reader ${PROJECT_SOURCE_DIR}/test_drivers/utf8reader/utf8reader.cpp)
add_executable(docsample ${PROJECT_SOURCE_DIR}/samples/docsample.cpp)
enable_testing()
add_test(smoke_test smoke)
add_test(negative_test negative ${PROJECT_SOURCE_DIR}/test_data/negative/utf8_invalid.txt)

1090
lib/utfcpp-2.3.5/README.md Normal file
View File

File diff suppressed because it is too large Load Diff

52
lib/utfcpp-2.3.5/samples/docsample.cpp Normal file
View File

@ -0,0 +1,52 @@
#include "../source/utf8.h"
#include <iostream>
#include <fstream>
#include <string>
#include <vector>
using namespace std;
int main(int argc, char** argv)
{
if (argc != 2) {
cout << "\nUsage: docsample filename\n";
return 0;
}
const char* test_file_path = argv[1];
// Open the test file (must be UTF-8 encoded)
ifstream fs8(test_file_path);
if (!fs8.is_open()) {
cout << "Could not open " << test_file_path << endl;
return 0;
}
unsigned line_count = 1;
string line;
// Play with all the lines in the file
while (getline(fs8, line)) {
// check for invalid utf-8 (for a simple yes/no check, there is also utf8::is_valid function)
string::iterator end_it = utf8::find_invalid(line.begin(), line.end());
if (end_it != line.end()) {
cout << "Invalid UTF-8 encoding detected at line " << line_count << "\n";
cout << "This part is fine: " << string(line.begin(), end_it) << "\n";
}
// Get the line length (at least for the valid part)
int length = utf8::distance(line.begin(), end_it);
cout << "Length of line " << line_count << " is " << length << "\n";
// Convert it to utf-16
vector<unsigned short> utf16line;
utf8::utf8to16(line.begin(), end_it, back_inserter(utf16line));
// And back to utf-8;
string utf8line;
utf8::utf16to8(utf16line.begin(), utf16line.end(), back_inserter(utf8line));
// Confirm that the conversion went OK:
if (utf8line != string(line.begin(), end_it))
cout << "Error in UTF-16 conversion at line: " << line_count << "\n";
line_count++;
}
return 0;
}

34
lib/utfcpp-2.3.5/source/utf8.h Normal file
View File

@ -0,0 +1,34 @@
// Copyright 2006 Nemanja Trifunovic
/*
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
#include "utf8/checked.h"
#include "utf8/unchecked.h"
#endif // header guard

327
lib/utfcpp-2.3.5/source/utf8/checked.h Normal file
View File

@ -0,0 +1,327 @@
// Copyright 2006-2016 Nemanja Trifunovic
/*
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
#define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
#include "core.h"
#include <stdexcept>
namespace utf8
{
// Base for the exceptions that may be thrown from the library
class exception : public ::std::exception {
};
// Exceptions that may be thrown from the library functions.
class invalid_code_point : public exception {
uint32_t cp;
public:
invalid_code_point(uint32_t codepoint) : cp(codepoint) {}
virtual const char* what() const throw() { return "Invalid code point"; }
uint32_t code_point() const {return cp;}
};
class invalid_utf8 : public exception {
uint8_t u8;
public:
invalid_utf8 (uint8_t u) : u8(u) {}
virtual const char* what() const throw() { return "Invalid UTF-8"; }
uint8_t utf8_octet() const {return u8;}
};
class invalid_utf16 : public exception {
uint16_t u16;
public:
invalid_utf16 (uint16_t u) : u16(u) {}
virtual const char* what() const throw() { return "Invalid UTF-16"; }
uint16_t utf16_word() const {return u16;}
};
class not_enough_room : public exception {
public:
virtual const char* what() const throw() { return "Not enough space"; }
};
/// The library API - functions intended to be called by the users
template <typename octet_iterator>
octet_iterator append(uint32_t cp, octet_iterator result)
{
if (!utf8::internal::is_code_point_valid(cp))
throw invalid_code_point(cp);
if (cp < 0x80) // one octet
*(result++) = static_cast<uint8_t>(cp);
else if (cp < 0x800) { // two octets
*(result++) = static_cast<uint8_t>((cp >> 6) | 0xc0);
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
}
else if (cp < 0x10000) { // three octets
*(result++) = static_cast<uint8_t>((cp >> 12) | 0xe0);
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
}
else { // four octets
*(result++) = static_cast<uint8_t>((cp >> 18) | 0xf0);
*(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f) | 0x80);
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
}
return result;
}
template <typename octet_iterator, typename output_iterator>
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement)
{
while (start != end) {
octet_iterator sequence_start = start;
internal::utf_error err_code = utf8::internal::validate_next(start, end);
switch (err_code) {
case internal::UTF8_OK :
for (octet_iterator it = sequence_start; it != start; ++it)
*out++ = *it;
break;
case internal::NOT_ENOUGH_ROOM:
throw not_enough_room();
case internal::INVALID_LEAD:
out = utf8::append (replacement, out);
++start;
break;
case internal::INCOMPLETE_SEQUENCE:
case internal::OVERLONG_SEQUENCE:
case internal::INVALID_CODE_POINT:
out = utf8::append (replacement, out);
++start;
// just one replacement mark for the sequence
while (start != end && utf8::internal::is_trail(*start))
++start;
break;
}
}
return out;
}
template <typename octet_iterator, typename output_iterator>
inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out)
{
static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd);
return utf8::replace_invalid(start, end, out, replacement_marker);
}
template <typename octet_iterator>
uint32_t next(octet_iterator& it, octet_iterator end)
{
uint32_t cp = 0;
internal::utf_error err_code = utf8::internal::validate_next(it, end, cp);
switch (err_code) {
case internal::UTF8_OK :
break;
case internal::NOT_ENOUGH_ROOM :
throw not_enough_room();
case internal::INVALID_LEAD :
case internal::INCOMPLETE_SEQUENCE :
case internal::OVERLONG_SEQUENCE :
throw invalid_utf8(*it);
case internal::INVALID_CODE_POINT :
throw invalid_code_point(cp);
}
return cp;
}
template <typename octet_iterator>
uint32_t peek_next(octet_iterator it, octet_iterator end)
{
return utf8::next(it, end);
}
template <typename octet_iterator>
uint32_t prior(octet_iterator& it, octet_iterator start)
{
// can't do much if it == start
if (it == start)
throw not_enough_room();
octet_iterator end = it;
// Go back until we hit either a lead octet or start
while (utf8::internal::is_trail(*(--it)))
if (it == start)
throw invalid_utf8(*it); // error - no lead byte in the sequence
return utf8::peek_next(it, end);
}
/// Deprecated in versions that include "prior"
template <typename octet_iterator>
uint32_t previous(octet_iterator& it, octet_iterator pass_start)
{
octet_iterator end = it;
while (utf8::internal::is_trail(*(--it)))
if (it == pass_start)
throw invalid_utf8(*it); // error - no lead byte in the sequence
octet_iterator temp = it;
return utf8::next(temp, end);
}
template <typename octet_iterator, typename distance_type>
void advance (octet_iterator& it, distance_type n, octet_iterator end)
{
for (distance_type i = 0; i < n; ++i)
utf8::next(it, end);
}
template <typename octet_iterator>
typename std::iterator_traits<octet_iterator>::difference_type
distance (octet_iterator first, octet_iterator last)
{
typename std::iterator_traits<octet_iterator>::difference_type dist;
for (dist = 0; first < last; ++dist)
utf8::next(first, last);
return dist;
}
template <typename u16bit_iterator, typename octet_iterator>
octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
{
while (start != end) {
uint32_t cp = utf8::internal::mask16(*start++);
// Take care of surrogate pairs first
if (utf8::internal::is_lead_surrogate(cp)) {
if (start != end) {
uint32_t trail_surrogate = utf8::internal::mask16(*start++);
if (utf8::internal::is_trail_surrogate(trail_surrogate))
cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
else
throw invalid_utf16(static_cast<uint16_t>(trail_surrogate));
}
else
throw invalid_utf16(static_cast<uint16_t>(cp));
}
// Lone trail surrogate
else if (utf8::internal::is_trail_surrogate(cp))
throw invalid_utf16(static_cast<uint16_t>(cp));
result = utf8::append(cp, result);
}
return result;
}
template <typename u16bit_iterator, typename octet_iterator>
u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
{
while (start < end) {
uint32_t cp = utf8::next(start, end);
if (cp > 0xffff) { //make a surrogate pair
*result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
*result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
}
else
*result++ = static_cast<uint16_t>(cp);
}
return result;
}
template <typename octet_iterator, typename u32bit_iterator>
octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
{
while (start != end)
result = utf8::append(*(start++), result);
return result;
}
template <typename octet_iterator, typename u32bit_iterator>
u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
{
while (start < end)
(*result++) = utf8::next(start, end);
return result;
}
// The iterator class
template <typename octet_iterator>
class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> {
octet_iterator it;
octet_iterator range_start;
octet_iterator range_end;
public:
iterator () {}
explicit iterator (const octet_iterator& octet_it,
const octet_iterator& rangestart,
const octet_iterator& rangeend) :
it(octet_it), range_start(rangestart), range_end(rangeend)
{
if (it < range_start || it > range_end)
throw std::out_of_range("Invalid utf-8 iterator position");
}
// the default "big three" are OK
octet_iterator base () const { return it; }
uint32_t operator * () const
{
octet_iterator temp = it;
return utf8::next(temp, range_end);
}
bool operator == (const iterator& rhs) const
{
if (range_start != rhs.range_start || range_end != rhs.range_end)
throw std::logic_error("Comparing utf-8 iterators defined with different ranges");
return (it == rhs.it);
}
bool operator != (const iterator& rhs) const
{
return !(operator == (rhs));
}
iterator& operator ++ ()
{
utf8::next(it, range_end);
return *this;
}
iterator operator ++ (int)
{
iterator temp = *this;
utf8::next(it, range_end);
return temp;
}
iterator& operator -- ()
{
utf8::prior(it, range_start);
return *this;
}
iterator operator -- (int)
{
iterator temp = *this;
utf8::prior(it, range_start);
return temp;
}
}; // class iterator
} // namespace utf8
#endif //header guard

332
lib/utfcpp-2.3.5/source/utf8/core.h Normal file
View File

@ -0,0 +1,332 @@
// Copyright 2006 Nemanja Trifunovic
/*
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
#include <iterator>
namespace utf8
{
// The typedefs for 8-bit, 16-bit and 32-bit unsigned integers
// You may need to change them to match your system.
// These typedefs have the same names as ones from cstdint, or boost/cstdint
typedef unsigned char uint8_t;
typedef unsigned short uint16_t;
typedef unsigned int uint32_t;
// Helper code - not intended to be directly called by the library users. May be changed at any time
namespace internal
{
// Unicode constants
// Leading (high) surrogates: 0xd800 - 0xdbff
// Trailing (low) surrogates: 0xdc00 - 0xdfff
const uint16_t LEAD_SURROGATE_MIN = 0xd800u;
const uint16_t LEAD_SURROGATE_MAX = 0xdbffu;
const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u;
const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu;
const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10);
const uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN;
// Maximum valid value for a Unicode code point
const uint32_t CODE_POINT_MAX = 0x0010ffffu;
template<typename octet_type>
inline uint8_t mask8(octet_type oc)
{
return static_cast<uint8_t>(0xff & oc);
}
template<typename u16_type>
inline uint16_t mask16(u16_type oc)
{
return static_cast<uint16_t>(0xffff & oc);
}
template<typename octet_type>
inline bool is_trail(octet_type oc)
{
return ((utf8::internal::mask8(oc) >> 6) == 0x2);
}
template <typename u16>
inline bool is_lead_surrogate(u16 cp)
{
return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX);
}
template <typename u16>
inline bool is_trail_surrogate(u16 cp)
{
return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
}
template <typename u16>
inline bool is_surrogate(u16 cp)
{
return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
}
template <typename u32>
inline bool is_code_point_valid(u32 cp)
{
return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp));
}
template <typename octet_iterator>
inline typename std::iterator_traits<octet_iterator>::difference_type
sequence_length(octet_iterator lead_it)
{
uint8_t lead = utf8::internal::mask8(*lead_it);
if (lead < 0x80)
return 1;
else if ((lead >> 5) == 0x6)
return 2;
else if ((lead >> 4) == 0xe)
return 3;
else if ((lead >> 3) == 0x1e)
return 4;
else
return 0;
}
template <typename octet_difference_type>
inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length)
{
if (cp < 0x80) {
if (length != 1)
return true;
}
else if (cp < 0x800) {
if (length != 2)
return true;
}
else if (cp < 0x10000) {
if (length != 3)
return true;
}
return false;
}
enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT};
/// Helper for get_sequence_x
template <typename octet_iterator>
utf_error increase_safely(octet_iterator& it, octet_iterator end)
{
if (++it == end)
return NOT_ENOUGH_ROOM;
if (!utf8::internal::is_trail(*it))
return INCOMPLETE_SEQUENCE;
return UTF8_OK;
}
#define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;}
/// get_sequence_x functions decode utf-8 sequences of the length x
template <typename octet_iterator>
utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point)
{
if (it == end)
return NOT_ENOUGH_ROOM;
code_point = utf8::internal::mask8(*it);
return UTF8_OK;
}
template <typename octet_iterator>
utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point)
{
if (it == end)
return NOT_ENOUGH_ROOM;
code_point = utf8::internal::mask8(*it);
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f);
return UTF8_OK;
}
template <typename octet_iterator>
utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point)
{
if (it == end)
return NOT_ENOUGH_ROOM;
code_point = utf8::internal::mask8(*it);
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
code_point += (*it) & 0x3f;
return UTF8_OK;
}
template <typename octet_iterator>
utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point)
{
if (it == end)
return NOT_ENOUGH_ROOM;
code_point = utf8::internal::mask8(*it);
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
code_point += (utf8::internal::mask8(*it) << 6) & 0xfff;
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
code_point += (*it) & 0x3f;
return UTF8_OK;
}
#undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR
template <typename octet_iterator>
utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point)
{
if (it == end)
return NOT_ENOUGH_ROOM;
// Save the original value of it so we can go back in case of failure
// Of course, it does not make much sense with i.e. stream iterators
octet_iterator original_it = it;
uint32_t cp = 0;
// Determine the sequence length based on the lead octet
typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type;
const octet_difference_type length = utf8::internal::sequence_length(it);
// Get trail octets and calculate the code point
utf_error err = UTF8_OK;
switch (length) {
case 0:
return INVALID_LEAD;
case 1:
err = utf8::internal::get_sequence_1(it, end, cp);
break;
case 2:
err = utf8::internal::get_sequence_2(it, end, cp);
break;
case 3:
err = utf8::internal::get_sequence_3(it, end, cp);
break;
case 4:
err = utf8::internal::get_sequence_4(it, end, cp);
break;
}
if (err == UTF8_OK) {
// Decoding succeeded. Now, security checks...
if (utf8::internal::is_code_point_valid(cp)) {
if (!utf8::internal::is_overlong_sequence(cp, length)){
// Passed! Return here.
code_point = cp;
++it;
return UTF8_OK;
}
else
err = OVERLONG_SEQUENCE;
}
else
err = INVALID_CODE_POINT;
}
// Failure branch - restore the original value of the iterator
it = original_it;
return err;
}
template <typename octet_iterator>
inline utf_error validate_next(octet_iterator& it, octet_iterator end) {
uint32_t ignored;
return utf8::internal::validate_next(it, end, ignored);
}
} // namespace internal
/// The library API - functions intended to be called by the users
// Byte order mark
const uint8_t bom[] = {0xef, 0xbb, 0xbf};
template <typename octet_iterator>
octet_iterator find_invalid(octet_iterator start, octet_iterator end)
{
octet_iterator result = start;
while (result != end) {
utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end);
if (err_code != internal::UTF8_OK)
return result;
}
return result;
}
template <typename octet_iterator>
inline bool is_valid(octet_iterator start, octet_iterator end)
{
return (utf8::find_invalid(start, end) == end);
}
template <typename octet_iterator>
inline bool starts_with_bom (octet_iterator it, octet_iterator end)
{
return (
((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) &&
((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) &&
((it != end) && (utf8::internal::mask8(*it)) == bom[2])
);
}
//Deprecated in release 2.3
template <typename octet_iterator>
inline bool is_bom (octet_iterator it)
{
return (
(utf8::internal::mask8(*it++)) == bom[0] &&
(utf8::internal::mask8(*it++)) == bom[1] &&
(utf8::internal::mask8(*it)) == bom[2]
);
}
} // namespace utf8
#endif // header guard

228
lib/utfcpp-2.3.5/source/utf8/unchecked.h Normal file
View File

@ -0,0 +1,228 @@
// Copyright 2006 Nemanja Trifunovic
/*
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:
The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/
#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
#include "core.h"
namespace utf8
{
namespace unchecked
{
template <typename octet_iterator>
octet_iterator append(uint32_t cp, octet_iterator result)
{
if (cp < 0x80) // one octet
*(result++) = static_cast<uint8_t>(cp);
else if (cp < 0x800) { // two octets
*(result++) = static_cast<uint8_t>((cp >> 6) | 0xc0);
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
}
else if (cp < 0x10000) { // three octets
*(result++) = static_cast<uint8_t>((cp >> 12) | 0xe0);
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
}
else { // four octets
*(result++) = static_cast<uint8_t>((cp >> 18) | 0xf0);
*(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f)| 0x80);
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
}
return result;
}
template <typename octet_iterator>
uint32_t next(octet_iterator& it)
{
uint32_t cp = utf8::internal::mask8(*it);
typename std::iterator_traits<octet_iterator>::difference_type length = utf8::internal::sequence_length(it);
switch (length) {
case 1:
break;
case 2:
it++;
cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
break;
case 3:
++it;
cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
++it;
cp += (*it) & 0x3f;
break;
case 4:
++it;
cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
++it;
cp += (utf8::internal::mask8(*it) << 6) & 0xfff;
++it;
cp += (*it) & 0x3f;
break;
}
++it;
return cp;
}
template <typename octet_iterator>
uint32_t peek_next(octet_iterator it)
{
return utf8::unchecked::next(it);
}
template <typename octet_iterator>
uint32_t prior(octet_iterator& it)
{
while (utf8::internal::is_trail(*(--it))) ;
octet_iterator temp = it;
return utf8::unchecked::next(temp);
}
// Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous)
template <typename octet_iterator>
inline uint32_t previous(octet_iterator& it)
{
return utf8::unchecked::prior(it);
}
template <typename octet_iterator, typename distance_type>
void advance (octet_iterator& it, distance_type n)
{
for (distance_type i = 0; i < n; ++i)
utf8::unchecked::next(it);
}
template <typename octet_iterator>
typename std::iterator_traits<octet_iterator>::difference_type
distance (octet_iterator first, octet_iterator last)
{
typename std::iterator_traits<octet_iterator>::difference_type dist;
for (dist = 0; first < last; ++dist)
utf8::unchecked::next(first);
return dist;
}
template <typename u16bit_iterator, typename octet_iterator>
octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
{
while (start != end) {
uint32_t cp = utf8::internal::mask16(*start++);
// Take care of surrogate pairs first
if (utf8::internal::is_lead_surrogate(cp)) {
uint32_t trail_surrogate = utf8::internal::mask16(*start++);
cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
}
result = utf8::unchecked::append(cp, result);
}
return result;
}
template <typename u16bit_iterator, typename octet_iterator>
u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
{
while (start < end) {
uint32_t cp = utf8::unchecked::next(start);
if (cp > 0xffff) { //make a surrogate pair
*result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
*result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
}
else
*result++ = static_cast<uint16_t>(cp);
}
return result;
}
template <typename octet_iterator, typename u32bit_iterator>
octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
{
while (start != end)
result = utf8::unchecked::append(*(start++), result);
return result;
}
template <typename octet_iterator, typename u32bit_iterator>
u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
{
while (start < end)
(*result++) = utf8::unchecked::next(start);
return result;
}
// The iterator class
template <typename octet_iterator>
class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> {
octet_iterator it;
public:
iterator () {}
explicit iterator (const octet_iterator& octet_it): it(octet_it) {}
// the default "big three" are OK
octet_iterator base () const { return it; }
uint32_t operator * () const
{
octet_iterator temp = it;
return utf8::unchecked::next(temp);
}
bool operator == (const iterator& rhs) const
{
return (it == rhs.it);
}
bool operator != (const iterator& rhs) const
{
return !(operator == (rhs));
}
iterator& operator ++ ()
{
::std::advance(it, utf8::internal::sequence_length(it));
return *this;
}
iterator operator ++ (int)
{
iterator temp = *this;
::std::advance(it, utf8::internal::sequence_length(it));
return temp;
}
iterator& operator -- ()
{
utf8::unchecked::prior(it);
return *this;
}
iterator operator -- (int)
{
iterator temp = *this;
utf8::unchecked::prior(it);
return temp;
}
}; // class iterator
} // namespace utf8::unchecked
} // namespace utf8
#endif // header guard

BIN
lib/utfcpp-2.3.5/test_data/negative/utf8_invalid.txt Normal file
View File

Binary file not shown.

212
lib/utfcpp-2.3.5/test_data/utf8samples/UTF-8-demo.txt Normal file
View File

@ -0,0 +1,212 @@
UTF-8 encoded sample plain-text file
‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
Markus Kuhn [ˈmaʳkʊs kuːn] <http://www.cl.cam.ac.uk/~mgk25/> — 2002-07-25
The ASCII compatible UTF-8 encoding used in this plain-text file
is defined in Unicode, ISO 10646-1, and RFC 2279.
Using Unicode/UTF-8, you can write in emails and source code things such as
Mathematics and sciences:
∮ E⋅da = Q, n → ∞, ∑ f(i) = ∏ g(i), ⎧⎡⎛┌─────┐⎞⎤⎫
⎪⎢⎜│a²+b³ ⎟⎥⎪
∀x∈ℝ: ⌈x⌉ = −⌊−x⌋, α ∧ ¬β = ¬(¬α ∨ β), ⎪⎢⎜│───── ⎟⎥⎪
⎪⎢⎜⎷ c₈ ⎟⎥⎪
ℕ ⊆ ℕ₀ ⊂ ℤ ⊂ ℚ ⊂ ℝ ⊂ ℂ, ⎨⎢⎜ ⎟⎥⎬
⎪⎢⎜ ∞ ⎟⎥⎪
⊥ < a ≠ b ≡ c ≤ d ≪ ⊤ ⇒ (⟦A⟧ ⇔ ⟪B⟫), ⎪⎢⎜ ⎲ ⎟⎥⎪
⎪⎢⎜ ⎳aⁱ-bⁱ⎟⎥⎪
2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm ⎩⎣⎝i=1 ⎠⎦⎭
Linguistics and dictionaries:
ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeıʃn
Y [ˈʏpsilɔn], Yen [jɛn], Yoga [ˈjoːgɑ]
APL:
((V⍳V)=⍳⍴V)/V←,V ⌷←⍳→⍴∆∇⊃‾⍎⍕⌈
Nicer typography in plain text files:
╔══════════════════════════════════════════╗
║ ║
║ • ‘single’ and “double” quotes ║
║ ║
║ • Curly apostrophes: “We’ve been here” ║
║ ║
║ • Latin-1 apostrophe and accents: '´` ║
║ ║
║ • ‚deutsche‘ „Anführungszeichen“ ║
║ ║
║ • †, ‡, ‰, •, 3–4, —, −5/+5, ™, … ║
║ ║
║ • ASCII safety test: 1lI|, 0OD, 8B ║
║ ╭─────────╮ ║
║ • the euro symbol: │ 14.95 € │ ║
║ ╰─────────╯ ║
╚══════════════════════════════════════════╝
Combining characters:
STARGΛ̊TE SG-1, a = v̇ = r̈, a⃑ ⊥ b⃑
Greek (in Polytonic):
The Greek anthem:
Σὲ γνωρίζω ἀπὸ τὴν κόψη
τοῦ σπαθιοῦ τὴν τρομερή,
σὲ γνωρίζω ἀπὸ τὴν ὄψη
ποὺ μὲ βία μετράει τὴ γῆ.
᾿Απ᾿ τὰ κόκκαλα βγαλμένη
τῶν ῾Ελλήνων τὰ ἱερά
καὶ σὰν πρῶτα ἀνδρειωμένη
χαῖρε, ὦ χαῖρε, ᾿Ελευθεριά!
From a speech of Demosthenes in the 4th century BC:
Οὐχὶ ταὐτὰ παρίσταταί μοι γιγνώσκειν, ὦ ἄνδρες ᾿Αθηναῖοι,
ὅταν τ᾿ εἰς τὰ πράγματα ἀποβλέψω καὶ ὅταν πρὸς τοὺς
λόγους οὓς ἀκούω· τοὺς μὲν γὰρ λόγους περὶ τοῦ
τιμωρήσασθαι Φίλιππον ὁρῶ γιγνομένους, τὰ δὲ πράγματ᾿
εἰς τοῦτο προήκοντα, ὥσθ᾿ ὅπως μὴ πεισόμεθ᾿ αὐτοὶ
πρότερον κακῶς σκέψασθαι δέον. οὐδέν οὖν ἄλλο μοι δοκοῦσιν
οἱ τὰ τοιαῦτα λέγοντες ἢ τὴν ὑπόθεσιν, περὶ ἧς βουλεύεσθαι,
οὐχὶ τὴν οὖσαν παριστάντες ὑμῖν ἁμαρτάνειν. ἐγὼ δέ, ὅτι μέν
ποτ᾿ ἐξῆν τῇ πόλει καὶ τὰ αὑτῆς ἔχειν ἀσφαλῶς καὶ Φίλιππον
τιμωρήσασθαι, καὶ μάλ᾿ ἀκριβῶς οἶδα· ἐπ᾿ ἐμοῦ γάρ, οὐ πάλαι
γέγονεν ταῦτ᾿ ἀμφότερα· νῦν μέντοι πέπεισμαι τοῦθ᾿ ἱκανὸν
προλαβεῖν ἡμῖν εἶναι τὴν πρώτην, ὅπως τοὺς συμμάχους
σώσομεν. ἐὰν γὰρ τοῦτο βεβαίως ὑπάρξῃ, τότε καὶ περὶ τοῦ
τίνα τιμωρήσεταί τις καὶ ὃν τρόπον ἐξέσται σκοπεῖν· πρὶν δὲ
τὴν ἀρχὴν ὀρθῶς ὑποθέσθαι, μάταιον ἡγοῦμαι περὶ τῆς
τελευτῆς ὁντινοῦν ποιεῖσθαι λόγον.
Δημοσθένους, Γ´ ᾿Ολυνθιακὸς
Georgian:
From a Unicode conference invitation:
გთხოვთ ახლავე გაიაროთ რეგისტრაცია Unicode-ის მეათე საერთაშორისო
კონფერენციაზე დასასწრებად, რომელიც გაიმართება 10-12 მარტს,
ქ. მაინცში, გერმანიაში. კონფერენცია შეჰკრებს ერთად მსოფლიოს
ექსპერტებს ისეთ დარგებში როგორიცაა ინტერნეტი და Unicode-ი,
ინტერნაციონალიზაცია და ლოკალიზაცია, Unicode-ის გამოყენება
ოპერაციულ სისტემებსა, და გამოყენებით პროგრამებში, შრიფტებში,
ტექსტების დამუშავებასა და მრავალენოვან კომპიუტერულ სისტემებში.
Russian:
From a Unicode conference invitation:
Зарегистрируйтесь сейчас на Десятую Международную Конференцию по
Unicode, которая состоится 10-12 марта 1997 года в Майнце в Германии.
Конференция соберет широкий круг экспертов по вопросам глобального
Интернета и Unicode, локализации и интернационализации, воплощению и
применению Unicode в различных операционных системах и программных
приложениях, шрифтах, верстке и многоязычных компьютерных системах.
Thai (UCS Level 2):
Excerpt from a poetry on The Romance of The Three Kingdoms (a Chinese
classic 'San Gua'):
[----------------------------|------------------------]
๏ แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช พระปกเกศกองบู๊กู้ขึ้นใหม่
สิบสองกษัตริย์ก่อนหน้าแลถัดไป สององค์ไซร้โง่เขลาเบาปัญญา
ทรงนับถือขันทีเป็นที่พึ่ง บ้านเมืองจึงวิปริตเป็นนักหนา
โฮจิ๋นเรียกทัพทั่วหัวเมืองมา หมายจะฆ่ามดชั่วตัวสำคัญ
เหมือนขับไสไล่เสือจากเคหา รับหมาป่าเข้ามาเลยอาสัญ
ฝ่ายอ้องอุ้นยุแยกให้แตกกัน ใช้สาวนั้นเป็นชนวนชื่นชวนใจ
พลันลิฉุยกุยกีกลับก่อเหตุ ช่างอาเพศจริงหนาฟ้าร้องไห้
ต้องรบราฆ่าฟันจนบรรลัย ฤๅหาใครค้ำชูกู้บรรลังก์ ฯ
(The above is a two-column text. If combining characters are handled
correctly, the lines of the second column should be aligned with the
| character above.)
Ethiopian:
Proverbs in the Amharic language:
ሰማይ አይታረስ ንጉሥ አይከሰስ።
ብላ ካለኝ እንደአባቴ በቆመጠኝ።
ጌጥ ያለቤቱ ቁምጥና ነው።
ደሀ በሕልሙ ቅቤ ባይጠጣ ንጣት በገደለው።
የአፍ ወለምታ በቅቤ አይታሽም።
አይጥ በበላ ዳዋ ተመታ።
ሲተረጉሙ ይደረግሙ።
ቀስ በቀስ፥ ዕንቁላል በእግሩ ይሄዳል።
ድር ቢያብር አንበሳ ያስር።
ሰው እንደቤቱ እንጅ እንደ ጉረቤቱ አይተዳደርም።
እግዜር የከፈተውን ጉሮሮ ሳይዘጋው አይድርም።
የጎረቤት ሌባ፥ ቢያዩት ይስቅ ባያዩት ያጠልቅ።
ሥራ ከመፍታት ልጄን ላፋታት።
ዓባይ ማደሪያ የለው፥ ግንድ ይዞ ይዞራል።
የእስላም አገሩ መካ የአሞራ አገሩ ዋርካ።
ተንጋሎ ቢተፉ ተመልሶ ባፉ።
ወዳጅህ ማር ቢሆን ጨርስህ አትላሰው።
እግርህን በፍራሽህ ልክ ዘርጋ።
Runes:
ᚻᛖ ᚳᚹᚫᚦ ᚦᚫᛏ ᚻᛖ ᛒᚢᛞᛖ ᚩᚾ ᚦᚫᛗ ᛚᚪᚾᛞᛖ ᚾᚩᚱᚦᚹᛖᚪᚱᛞᚢᛗ ᚹᛁᚦ ᚦᚪ ᚹᛖᛥᚫ
(Old English, which transcribed into Latin reads 'He cwaeth that he
bude thaem lande northweardum with tha Westsae.' and means 'He said
that he lived in the northern land near the Western Sea.')
Braille:
⡌⠁⠧⠑ ⠼⠁⠒ ⡍⠜⠇⠑⠹⠰⠎ ⡣⠕⠌
⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠙⠑⠁⠙⠒ ⠞⠕ ⠃⠑⠛⠔ ⠺⠊⠹⠲ ⡹⠻⠑ ⠊⠎ ⠝⠕ ⠙⠳⠃⠞
⠱⠁⠞⠑⠧⠻ ⠁⠃⠳⠞ ⠹⠁⠞⠲ ⡹⠑ ⠗⠑⠛⠊⠌⠻ ⠕⠋ ⠙⠊⠎ ⠃⠥⠗⠊⠁⠇ ⠺⠁⠎
⠎⠊⠛⠝⠫ ⠃⠹ ⠹⠑ ⠊⠇⠻⠛⠹⠍⠁⠝⠂ ⠹⠑ ⠊⠇⠻⠅⠂ ⠹⠑ ⠥⠝⠙⠻⠞⠁⠅⠻⠂
⠁⠝⠙ ⠹⠑ ⠡⠊⠑⠋ ⠍⠳⠗⠝⠻⠲ ⡎⠊⠗⠕⠕⠛⠑ ⠎⠊⠛⠝⠫ ⠊⠞⠲ ⡁⠝⠙
⡎⠊⠗⠕⠕⠛⠑⠰⠎ ⠝⠁⠍⠑ ⠺⠁⠎ ⠛⠕⠕⠙ ⠥⠏⠕⠝ ⠰⡡⠁⠝⠛⠑⠂ ⠋⠕⠗ ⠁⠝⠹⠹⠔⠛ ⠙⠑
⠡⠕⠎⠑ ⠞⠕ ⠏⠥⠞ ⠙⠊⠎ ⠙⠁⠝⠙ ⠞⠕⠲
⡕⠇⠙ ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲
⡍⠔⠙⠖ ⡊ ⠙⠕⠝⠰⠞ ⠍⠑⠁⠝ ⠞⠕ ⠎⠁⠹ ⠹⠁⠞ ⡊ ⠅⠝⠪⠂ ⠕⠋ ⠍⠹
⠪⠝ ⠅⠝⠪⠇⠫⠛⠑⠂ ⠱⠁⠞ ⠹⠻⠑ ⠊⠎ ⠏⠜⠞⠊⠊⠥⠇⠜⠇⠹ ⠙⠑⠁⠙ ⠁⠃⠳⠞
⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲ ⡊ ⠍⠊⠣⠞ ⠙⠁⠧⠑ ⠃⠑⠲ ⠔⠊⠇⠔⠫⠂ ⠍⠹⠎⠑⠇⠋⠂ ⠞⠕
⠗⠑⠛⠜⠙ ⠁ ⠊⠕⠋⠋⠔⠤⠝⠁⠊⠇ ⠁⠎ ⠹⠑ ⠙⠑⠁⠙⠑⠌ ⠏⠊⠑⠊⠑ ⠕⠋ ⠊⠗⠕⠝⠍⠕⠝⠛⠻⠹
⠔ ⠹⠑ ⠞⠗⠁⠙⠑⠲ ⡃⠥⠞ ⠹⠑ ⠺⠊⠎⠙⠕⠍ ⠕⠋ ⠳⠗ ⠁⠝⠊⠑⠌⠕⠗⠎
⠊⠎ ⠔ ⠹⠑ ⠎⠊⠍⠊⠇⠑⠆ ⠁⠝⠙ ⠍⠹ ⠥⠝⠙⠁⠇⠇⠪⠫ ⠙⠁⠝⠙⠎
⠩⠁⠇⠇ ⠝⠕⠞ ⠙⠊⠌⠥⠗⠃ ⠊⠞⠂ ⠕⠗ ⠹⠑ ⡊⠳⠝⠞⠗⠹⠰⠎ ⠙⠕⠝⠑ ⠋⠕⠗⠲ ⡹⠳
⠺⠊⠇⠇ ⠹⠻⠑⠋⠕⠗⠑ ⠏⠻⠍⠊⠞ ⠍⠑ ⠞⠕ ⠗⠑⠏⠑⠁⠞⠂ ⠑⠍⠏⠙⠁⠞⠊⠊⠁⠇⠇⠹⠂ ⠹⠁⠞
⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲
(The first couple of paragraphs of "A Christmas Carol" by Dickens)
Compact font selection example text:
ABCDEFGHIJKLMNOPQRSTUVWXYZ /0123456789
abcdefghijklmnopqrstuvwxyz £©µÀÆÖÞßéöÿ
–—‘“”„†•…‰™œŠŸž€ ΑΒΓΔΩαβγδω АБВГДабвгд
∀∂∈ℝ∧∪≡∞ ↑↗↨↻⇣ ┐┼╔╘░►☺♀ fi<>⑀₂ἠḂӥẄɐː⍎אԱა
Greetings in various languages:
Hello world, Καλημέρα κόσμε, コンニチハ
Box drawing alignment tests: █
▉
╔══╦══╗ ┌──┬──┐ ╭──┬──╮ ╭──┬──╮ ┏━━┳━━┓ ┎┒┏┑ ╷ ╻ ┏┯┓ ┌┰┐ ▊ ╱╲╱╲╳╳╳
║┌─╨─┐║ │╔═╧═╗│ │╒═╪═╕│ │╓─╁─╖│ ┃┌─╂─┐┃ ┗╃╄┙ ╶┼╴╺╋╸┠┼┨ ┝╋┥ ▋ ╲╱╲╱╳╳╳
║│╲ ╱│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ ╿ │┃ ┍╅╆┓ ╵ ╹ ┗┷┛ └┸┘ ▌ ╱╲╱╲╳╳╳
╠╡ ╳ ╞╣ ├╢ ╟┤ ├┼─┼─┼┤ ├╫─╂─╫┤ ┣┿╾┼╼┿┫ ┕┛┖┚ ┌┄┄┐ ╎ ┏┅┅┓ ┋ ▍ ╲╱╲╱╳╳╳
║│╱ ╲│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ ╽ │┃ ░░▒▒▓▓██ ┊ ┆ ╎ ╏ ┇ ┋ ▎
║└─╥─┘║ │╚═╤═╝│ │╘═╪═╛│ │╙─╀─╜│ ┃└─╂─┘┃ ░░▒▒▓▓██ ┊ ┆ ╎ ╏ ┇ ┋ ▏
╚══╩══╝ └──┴──┘ ╰──┴──╯ ╰──┴──╯ ┗━━┻━━┛ ▗▄▖▛▀▜ └╌╌┘ ╎ ┗╍╍┛ ┋ ▁▂▃▄▅▆▇█
▝▀▘▙▄▟

167
lib/utfcpp-2.3.5/test_data/utf8samples/Unicode_transcriptions.html Normal file
View File

@ -0,0 +1,167 @@
? *Unicode Transcriptions* Notes <#Notes>
Glyphs <http://www.macchiato.com/unicode/show.html> | Samples
<http://www.macchiato.com/unicode/Unicode_transcriptions.html> | Charts
<http://www.macchiato.com/unicode/charts.html> | UTF
<http://www.macchiato.com/unicode/convert.html> | Forms
<http://www-4.ibm.com/software/developer/library/utfencodingforms/> |
Home <http://www.macchiato.com>.
<http://member.linkexchange.com/cgi-bin/fc/fastcounter-login?750641>
Name Text Image
Arabic (Arabic) يونِكود ?
Arabic (Persian) یونی‌کُد / ?/
Armenian Յունիկօդ
Bengali য়ূনিকোড
Bopomofo ㄊㄨㄥ˅ ㄧˋ ㄇㄚ˅
ㄨㄢˋ ㄍㄨㄛˊ ㄇㄚ˅
Braille
Buhid
Canadian Aboriginal ᔫᗂᑰᑦ
Cherokee ᏳᏂᎪᏛ
Cypriot
Cyrillic (Russian) Юникод ?
Deseret (English) ???????
Devanagari (Hindi) यूनिकोड ?
Ethiopic ዩኒኮድ
Georgian უნიკოდი ?
Gothic
Greek Γιούνικοντ
Gujarati યૂનિકોડ
Gurmukhi ਯੂਨਿਕੋਡ
Han (Chinese) 统一码 ?
統一碼 ?
万国码 ?
萬國碼 ?
Hangul 유니코드
Hanunoo
Hebrew יוניקוד
Hebrew (pointed) יוּנִיקוׁד
Hebrew (Yiddish) יוניקאָד ?
Hiragana (Japanese) ゆにこおど
Katakana (Japanese) ユニコード ?
Kannada ಯೂನಿಕೋಡ್
Khmer យូនីគោដ
Lao
Latin Unicode Unicode
Latin (IPA <#English_Pronunciation>) ˈjunɪˌkoːd ?
Latin (Am. Dict. <#American_Dictionary>) Ūnĭcōde̽ ?
Limbu
Linear B
Malayalam യൂനികോഡ്
Mongolian
Myanmar
Ogham ᚔᚒᚅᚔᚉᚑᚇ / /
Old Italic
Oriya ୟୂନିକୋଡ
Osmanya
Runic (Anglo-Saxon) ᛡᚢᚾᛁᚳᚩᛞ
Shavian
Sinhala යණනිකෞද්
Syriac ܝܘܢܝܩܘܕ
Tagbanwa
Tagalog
Tai Le
Tamil யூனிகோட்
Telugu యూనికోడ్
Thaana
Thai ยูนืโคด
Tibetan (Dzongkha) ཨུ་ནི་ཀོཌྲ།
Ugaritic
Yi
Notes:
There are different ways to transcribe the word “Unicode”, depending on
the language and script. In some cases there is only one language that
customarily uses a given script; in others there are many languages. The
goal here is at a minimum to collect at least one transcription for each
script in a language customarily written in that script, with more
languages if possible. If the transcription is the same for multiple
languages in a script, then a single representative language is used.
Still missing are transcriptions for the items above in RED (in at least
one language). I would appreciate any other transcriptions, or
corrections for the ones listed here. Send to mark3@macchiato.com
<mailto:mark3@macchiato.com>, using the directions below:
* *Supplying Missing Items*
o Most Latin-script languages will follow the spelling, and
change the pronunciation. For any that would not, it would
be good to have the alternate spelling.
o For non-Latin scripts the goal is to match the English
pronunciation — /*not*/ spelling. Above is the IPA <#IPA>
(in phonemic transcription) that should be matched as
closely as possible (without sounding affected in the target
language)
o Text would be best in either the UTF-8 text, or the code
points in hex HTML. E.g. either of the following:
+ "Юникод"
+ "&#x042E;&#x043D;&#x0438;&#x043A;&#x043E;&#x0434;"
+ Note: for / supplementary characters/
<http://www.unicode.org/glossary/#supplementary_character>,
there should be one hex number per code point, not two
surrogates
<http://www.unicode.org/glossary/#surrogate_code_point>:
# &#x10000; /*not*/ &#xD800;&xDC00;
o If you have a good font, I'd also appreciate a GIF. It
should be *96 x 24* bits, with the text centered, in black
on white (plus grays if smoothed).
* *Other Comments*
o Because some browsers won't handle the text, both text and
GIF image are supplied. If you can’t read the text columns,
see Display Problems
<http://www.unicode.org/help/display_problems.html>.
o The Chinese versions (inc. Bopomofo) are translations, not
transcriptions, since "transcription in Chinese is pretty
lame" [J. Becker].
o There are other "translations" of Unicode that may be in
use, such as the Vietnamese "Thống Nhất Mã".
o For sample pages in different languages on the Unicode site,
see What is Unicode?
<http://www.unicode.org/unicode/standard/WhatIsUnicode.html>
o Americans are not generally used to IPA, and find a variety
of different systems in their dictionaries. This one leaves
the base letters as they are, and uses diacritics for
pronunciation.
* *Etymology of /Unicode/*
o Coined by J. Becker. Not related to previous usages, such as:
+ A telegraphic code in which one word or set of letters
represents a sentence or phrase; a telegram or message
in this. (late 19th century, OED)
o According to my references, the prefix "uni" is directly
from Latin while the word "code" is through French.
o The original Indo-European apparently would have been
*oino-kau-do ("one strike give"): *kau apparently being
related to such English words as: hew, haggle, hoe, hag,
hay, hack, caudad, caudal, caudate, caudex, coda, codex,
codicil, coward, incus, and Kovač (personal name: "smith").
+ I will leave the exact derivations to the exegetes,
but I like the association with "haggle" myself.
* *Contributions*
o This draws on contributions or comments from:
+ Dixon Au
+ Joe Becker
+ Maurice Bauhahn
+ Abel Cheung
+ Peter Constable
+ Michael Everson
+ Christopher John Fynn
+ Michael Kaplan
+ George Kiraz
+ Abdul Malik
+ Siva Nataraja
+ Roozbeh Pournader
+ Jonathan Rosenne
+ Jungshik Shin
------------------------------------------------------------------------
Terms of Use <http://www.macchiato.com/terms_of_use.html>. Last updated:
MED - 04/20/2003 15:30:33.
<http://member.linkexchange.com/cgi-bin/fc/fastcounter-login?750641>

126
lib/utfcpp-2.3.5/test_data/utf8samples/quickbrown.txt Normal file
View File

@ -0,0 +1,126 @@
Sentences that contain all letters commonly used in a language
--------------------------------------------------------------
Markus Kuhn <http://www.cl.cam.ac.uk/~mgk25/> -- 2001-09-02
This file is UTF-8 encoded.
Danish (da)
---------
Quizdeltagerne spiste jordbær med fløde, mens cirkusklovnen
Wolther spillede på xylofon.
(= Quiz contestants were eating strawbery with cream while Wolther
the circus clown played on xylophone.)
German (de)
-----------
Falsches Üben von Xylophonmusik quält jeden größeren Zwerg
(= Wrongful practicing of xylophone music tortures every larger dwarf)
Zwölf Boxkämpfer jagten Eva quer über den Sylter Deich
(= Twelve boxing fighters hunted Eva across the dike of Sylt)
Heizölrückstoßabdämpfung
(= fuel oil recoil absorber)
(jqvwxy missing, but all non-ASCII letters in one word)
English (en)
------------
The quick brown fox jumps over the lazy dog
Spanish (es)
------------
El pingüino Wenceslao hizo kilómetros bajo exhaustiva lluvia y
frío, añoraba a su querido cachorro.
(Contains every letter and every accent, but not every combination
of vowel + acute.)
French (fr)
-----------
Portez ce vieux whisky au juge blond qui fume sur son île intérieure, à
côté de l'alcôve ovoïde, où les bûches se consument dans l'âtre, ce
qui lui permet de penser à la cænogenèse de l'être dont il est question
dans la cause ambiguë entendue à Moÿ, dans un capharnaüm qui,
pense-t-il, diminue çà et là la qualité de son œuvre.
l'île exiguë
Où l'obèse jury mûr
Fête l'haï volapük,
Âne ex aéquo au whist,
Ôtez ce vœu déçu.
Le cœur déçu mais l'âme plutôt naïve, Louÿs rêva de crapaüter en
canoë au delà des îles, près du mälström où brûlent les novæ.
Irish Gaelic (ga)
-----------------
D'fhuascail Íosa, Úrmhac na hÓighe Beannaithe, pór Éava agus Ádhaimh
Hungarian (hu)
--------------
Árvíztűrő tükörfúrógép
(= flood-proof mirror-drilling machine, only all non-ASCII letters)
Icelandic (is)
--------------
Kæmi ný öxi hér ykist þjófum nú bæði víl og ádrepa
Sævör grét áðan því úlpan var ónýt
(some ASCII letters missing)
Japanese (jp)
-------------
Hiragana: (Iroha)
いろはにほへとちりぬるを
わかよたれそつねならむ
うゐのおくやまけふこえて
あさきゆめみしゑひもせす
Katakana:
イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム
ウヰノオクヤマ ケフコエテ アサキユメミシ ヱヒモセスン
Hebrew (iw)
-----------
? דג סקרן שט בים מאוכזב ולפתע מצא לו חברה איך הקליטה
Polish (pl)
-----------
Pchnąć w tę łódź jeża lub ośm skrzyń fig
(= To push a hedgehog or eight bins of figs in this boat)
Russian (ru)
------------
В чащах юга жил бы цитрус? Да, но фальшивый экземпляр!
(= Would a citrus live in the bushes of south? Yes, but only a fake one!)
Thai (th)
---------
[--------------------------|------------------------]
๏ เป็นมนุษย์สุดประเสริฐเลิศคุณค่า กว่าบรรดาฝูงสัตว์เดรัจฉาน
จงฝ่าฟันพัฒนาวิชาการ อย่าล้างผลาญฤๅเข่นฆ่าบีฑาใคร
ไม่ถือโทษโกรธแช่งซัดฮึดฮัดด่า หัดอภัยเหมือนกีฬาอัชฌาสัย
ปฏิบัติประพฤติกฎกำหนดใจ พูดจาให้จ๊ะๆ จ๋าๆ น่าฟังเอย ฯ
[The copyright for the Thai example is owned by The Computer
Association of Thailand under the Royal Patronage of His Majesty the
King.]
Please let me know if you find others! Special thanks to the people
from all over the world who contributed these sentences.

53
lib/utfcpp-2.3.5/test_drivers/negative/negative.cpp Normal file
View File

@ -0,0 +1,53 @@
#include "../../source/utf8.h"
using namespace utf8;
#include <string>
#include <iostream>
#include <fstream>
#include <algorithm>
using namespace std;
const unsigned INVALID_LINES[] = { 75, 76, 83, 84, 85, 93, 102, 103, 105, 106, 107, 108, 109, 110, 114, 115, 116, 117, 124, 125, 130, 135, 140, 145, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 169, 175, 176, 177, 207, 208, 209, 210, 211, 220, 221, 222, 223, 224, 232, 233, 234, 235, 236, 247, 248, 249, 250, 251, 252, 253, 257, 258, 259, 260, 261, 262, 263, 264};
const unsigned* INVALID_LINES_END = INVALID_LINES + sizeof(INVALID_LINES)/sizeof(unsigned);
int main(int argc, char** argv)
{
string test_file_path;
if (argc == 2)
test_file_path = argv[1];
else {
cout << "Wrong number of arguments" << endl;
exit(0);
}
// Open the test file
ifstream fs8(test_file_path.c_str());
if (!fs8.is_open()) {
cout << "Could not open " << test_file_path << endl;
return 0;
}
// Read it line by line
unsigned int line_count = 0;
char byte;
while (!fs8.eof()) {
string line;
while ((byte = static_cast<char>(fs8.get())) != '\n' && !fs8.eof())
line.push_back(byte);
line_count++;
bool expected_valid = (find(INVALID_LINES, INVALID_LINES_END, line_count) == INVALID_LINES_END);
// Print out lines that contain unexpected invalid UTF-8
if (!is_valid(line.begin(), line.end())) {
if (expected_valid)
cout << "Unexpected invalid utf-8 at line " << line_count << '\n';
// try fixing it:
string fixed_line;
replace_invalid(line.begin(), line.end(), back_inserter(fixed_line));
if (!is_valid(fixed_line.begin(), fixed_line.end()))
cout << "replace_invalid() resulted in an invalid utf-8 at line " << line_count << '\n';
}
else if (!expected_valid)
cout << "Invalid utf-8 NOT detected at line " << line_count << '\n';
}
}

297
lib/utfcpp-2.3.5/test_drivers/smoke_test/test.cpp Normal file
View File

@ -0,0 +1,297 @@
#include <cstring>
#include <cassert>
#include <vector>
#include "../../source/utf8.h"
using namespace utf8;
using namespace std;
int main()
{
//append
unsigned char u[5] = {0,0,0,0,0};
append(0x0448, u);
assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
append(0x65e5, u);
assert (u[0] == 0xe6 && u[1] == 0x97 && u[2] == 0xa5 && u[3] == 0 && u[4] == 0);
append(0x3044, u);
assert (u[0] == 0xe3 && u[1] == 0x81 && u[2] == 0x84 && u[3] == 0 && u[4] == 0);
append(0x10346, u);
assert (u[0] == 0xf0 && u[1] == 0x90 && u[2] == 0x8d && u[3] == 0x86 && u[4] == 0);
//next
const char* twochars = "\xe6\x97\xa5\xd1\x88";
const char* w = twochars;
int cp = next(w, twochars + 6);
assert (cp == 0x65e5);
assert (w == twochars + 3);
const char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
w = threechars;
cp = next(w, threechars + 9);
assert (cp == 0x10346);
assert (w == threechars + 4);
cp = next(w, threechars + 9);
assert (cp == 0x65e5);
assert (w == threechars + 7);
cp = next(w, threechars + 9);
assert (cp == 0x0448);
assert (w == threechars + 9);
//peek_next
const char* const cw = twochars;
cp = peek_next(cw, cw + 6);
assert (cp == 0x65e5);
assert (cw == twochars);
//prior
w = twochars + 3;
cp = prior (w, twochars);
assert (cp == 0x65e5);
assert (w == twochars);
w = threechars + 9;
cp = prior(w, threechars);
assert (cp == 0x0448);
assert (w == threechars + 7);
cp = prior(w, threechars);
assert (cp == 0x65e5);
assert (w == threechars + 4);
cp = prior(w, threechars);
assert (cp == 0x10346);
assert (w == threechars);
//previous (deprecated)
w = twochars + 3;
cp = previous (w, twochars - 1);
assert (cp == 0x65e5);
assert (w == twochars);
w = threechars + 9;
cp = previous(w, threechars - 1);
assert (cp == 0x0448);
assert (w == threechars + 7);
cp = previous(w, threechars -1);
assert (cp == 0x65e5);
assert (w == threechars + 4);
cp = previous(w, threechars - 1);
assert (cp == 0x10346);
assert (w == threechars);
// advance
w = twochars;
advance (w, 2, twochars + 6);
assert (w == twochars + 5);
// distance
size_t dist = utf8::distance(twochars, twochars + 5);
assert (dist == 2);
// utf32to8
int utf32string[] = {0x448, 0x65E5, 0x10346, 0};
vector<char> utf8result;
utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
assert (utf8result.size() == 9);
// try it with the return value;
char* utf8_end = utf32to8(utf32string, utf32string + 3, &utf8result[0]);
assert (utf8_end == &utf8result[0] + 9);
//utf8to32
vector<int> utf32result;
utf8to32(twochars, twochars + 5, back_inserter(utf32result));
assert (utf32result.size() == 2);
// try it with the return value;
int* utf32_end = utf8to32(twochars, twochars + 5, &utf32result[0]);
assert (utf32_end == &utf32result[0] + 2);
//utf16to8
unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
utf8result.clear();
utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
assert (utf8result.size() == 10);
// try it with the return value;
utf8_end = utf16to8 (utf16string, utf16string + 5, &utf8result[0]);
assert (utf8_end == &utf8result[0] + 10);
//utf8to16
char utf8_with_surrogates[] = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
vector <unsigned short> utf16result;
utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
assert (utf16result.size() == 4);
assert (utf16result[2] == 0xd834);
assert (utf16result[3] == 0xdd1e);
// try it with the return value;
unsigned short* utf16_end = utf8to16 (utf8_with_surrogates, utf8_with_surrogates + 9, &utf16result[0]);
assert (utf16_end == &utf16result[0] + 4);
//find_invalid
char utf_invalid[] = "\xe6\x97\xa5\xd1\x88\xfa";
char* invalid = find_invalid(utf_invalid, utf_invalid + 6);
assert (invalid == utf_invalid + 5);
//is_valid
bool bvalid = is_valid(utf_invalid, utf_invalid + 6);
assert (bvalid == false);
bvalid = is_valid(utf8_with_surrogates, utf8_with_surrogates + 9);
assert (bvalid == true);
//starts_with_bom
unsigned char byte_order_mark[] = {0xef, 0xbb, 0xbf};
bool bbom = starts_with_bom(byte_order_mark, byte_order_mark + sizeof(byte_order_mark));
assert (bbom == true);
bool no_bbom = starts_with_bom(threechars, threechars + sizeof(threechars));
assert (no_bbom == false);
//is_bom
bool unsafe_bbom = is_bom(byte_order_mark);
assert (unsafe_bbom == true);
//replace_invalid
char invalid_sequence[] = "a\x80\xe0\xa0\xc0\xaf\xed\xa0\x80z";
vector<char> replace_invalid_result(50);
replace_invalid (invalid_sequence, invalid_sequence + sizeof(invalid_sequence), replace_invalid_result.begin(), '?');
bvalid = is_valid(replace_invalid_result.begin(), replace_invalid_result.end());
assert (bvalid);
const char* fixed_invalid_sequence = "a????z";
assert (std::equal(replace_invalid_result.begin(), replace_invalid_result.begin() + sizeof(fixed_invalid_sequence), fixed_invalid_sequence));
// iterator
utf8::iterator<const char*> it(threechars, threechars, threechars + 9);
utf8::iterator<const char*> it2 = it;
assert (it2 == it);
assert (*it == 0x10346);
assert (*(++it) == 0x65e5);
assert ((*it++) == 0x65e5);
assert (*it == 0x0448);
assert (it != it2);
utf8::iterator<const char*> endit (threechars + 9, threechars, threechars + 9);
assert (++it == endit);
assert (*(--it) == 0x0448);
assert ((*it--) == 0x0448);
assert (*it == 0x65e5);
assert (--it == utf8::iterator<const char*>(threechars, threechars, threechars + 9));
assert (*it == 0x10346);
//////////////////////////////////////////////////////////
//// Unchecked variants
//////////////////////////////////////////////////////////
//append
memset(u, 0, 5);
append(0x0448, u);
assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
append(0x65e5, u);
assert (u[0] == 0xe6 && u[1] == 0x97 && u[2] == 0xa5 && u[3] == 0 && u[4] == 0);
append(0x10346, u);
assert (u[0] == 0xf0 && u[1] == 0x90 && u[2] == 0x8d && u[3] == 0x86 && u[4] == 0);
//next
w = twochars;
cp = unchecked::next(w);
assert (cp == 0x65e5);
assert (w == twochars + 3);
w = threechars;
cp = unchecked::next(w);
assert (cp == 0x10346);
assert (w == threechars + 4);
cp = unchecked::next(w);
assert (cp == 0x65e5);
assert (w == threechars + 7);
cp = unchecked::next(w);
assert (cp == 0x0448);
assert (w == threechars + 9);
//peek_next
cp = unchecked::peek_next(cw);
assert (cp == 0x65e5);
assert (cw == twochars);
//previous (calls prior internally)
w = twochars + 3;
cp = unchecked::previous (w);
assert (cp == 0x65e5);
assert (w == twochars);
w = threechars + 9;
cp = unchecked::previous(w);
assert (cp == 0x0448);
assert (w == threechars + 7);
cp = unchecked::previous(w);
assert (cp == 0x65e5);
assert (w == threechars + 4);
cp = unchecked::previous(w);
assert (cp == 0x10346);
assert (w == threechars);
// advance
w = twochars;
unchecked::advance (w, 2);
assert (w == twochars + 5);
// distance
dist = unchecked::distance(twochars, twochars + 5);
assert (dist == 2);
// utf32to8
utf8result.clear();
unchecked::utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
assert (utf8result.size() == 9);
// try it with the return value;
utf8_end = utf32to8(utf32string, utf32string + 3, &utf8result[0]);
assert(utf8_end == &utf8result[0] + 9);
//utf8to32
utf32result.clear();
unchecked::utf8to32(twochars, twochars + 5, back_inserter(utf32result));
assert (utf32result.size() == 2);
// try it with the return value;
utf32_end = utf8to32(twochars, twochars + 5, &utf32result[0]);
assert (utf32_end == &utf32result[0] + 2);
//utf16to8
utf8result.clear();
unchecked::utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
assert (utf8result.size() == 10);
// try it with the return value;
utf8_end = utf16to8 (utf16string, utf16string + 5, &utf8result[0]);
assert (utf8_end == &utf8result[0] + 10);
//utf8to16
utf16result.clear();
unchecked::utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
assert (utf16result.size() == 4);
assert (utf16result[2] == 0xd834);
assert (utf16result[3] == 0xdd1e);
// try it with the return value;
utf16_end = utf8to16 (utf8_with_surrogates, utf8_with_surrogates + 9, &utf16result[0]);
assert (utf16_end == &utf16result[0] + 4);
// iterator
utf8::unchecked::iterator<const char*> un_it(threechars);
utf8::unchecked::iterator<const char*> un_it2 = un_it;
assert (un_it2 == un_it);
assert (*un_it == 0x10346);
assert (*(++un_it) == 0x65e5);
assert ((*un_it++) == 0x65e5);
assert (un_it != un_it2);
assert (*un_it == 0x0448);
utf8::unchecked::iterator<const char*> un_endit (threechars + 9);
assert (++un_it == un_endit);
assert (*(--un_it) == 0x0448);
assert ((*un_it--) == 0x0448);
assert (*un_it == 0x65e5);
assert (--un_it == utf8::unchecked::iterator<const char*>(threechars));
assert (*un_it == 0x10346);
}

160
lib/utfcpp-2.3.5/test_drivers/utf8reader/utf8reader.cpp Normal file
View File

@ -0,0 +1,160 @@
#include "../../source/utf8.h"
using namespace utf8;
#include <string>
#include <iostream>
#include <fstream>
#include <vector>
using namespace std;
int main(int argc, char** argv)
{
if (argc != 2) {
cout << "\nUsage: utfreader filename\n";
return 0;
}
const char* TEST_FILE_PATH = argv[1];
// Open the test file
ifstream fs8(TEST_FILE_PATH);
if (!fs8.is_open()) {
cout << "Could not open " << TEST_FILE_PATH << endl;
return 0;
}
// Read it line by line
unsigned int line_count = 0;
char byte;
while (!fs8.eof()) {
string line;
while ((byte = static_cast<char>(fs8.get())) != '\n' && !fs8.eof())
line.push_back(byte);
line_count++;
// Play around with each line and convert it to utf16
string::iterator line_start = line.begin();
string::iterator line_end = line.end();
line_end = find_invalid(line_start, line_end);
if (line_end != line.end())
cout << "Line " << line_count << ": Invalid utf-8 at byte " << int(line.end() - line_end) << '\n';
// Convert it to utf-16 and write to the file
vector<unsigned short> utf16_line;
utf8to16(line_start, line_end, back_inserter(utf16_line));
// Back to utf-8 and compare it to the original line.
string back_to_utf8;
utf16to8(utf16_line.begin(), utf16_line.end(), back_inserter(back_to_utf8));
if (back_to_utf8.compare(string(line_start, line_end)) != 0)
cout << "Line " << line_count << ": Conversion to UTF-16 and back failed" << '\n';
// Now, convert it to utf-32, back to utf-8 and compare
vector <unsigned> utf32_line;
utf8to32(line_start, line_end, back_inserter(utf32_line));
back_to_utf8.clear();
utf32to8(utf32_line.begin(), utf32_line.end(), back_inserter(back_to_utf8));
if (back_to_utf8.compare(string(line_start, line_end)) != 0)
cout << "Line " << line_count << ": Conversion to UTF-32 and back failed" << '\n';
// Now, iterate and back
unsigned char_count = 0;
string::iterator it = line_start;
while (it != line_end) {
unsigned int next_cp = peek_next(it, line_end);
if (next(it, line_end) != next_cp)
cout << "Line " << line_count << ": Error: peek_next gave a different result than next" << '\n';
char_count++;
}
if (char_count != utf32_line.size())
cout << "Line " << line_count << ": Error in iterating with next - wrong number of characters" << '\n';
string::iterator adv_it = line_start;
utf8::advance(adv_it, char_count, line_end);
if (adv_it != line_end)
cout << "Line " << line_count << ": Error in advance function" << '\n';
if (string::size_type(utf8::distance(line_start, line_end)) != char_count)
cout << "Line " << line_count << ": Error in distance function" << '\n';
while (it != line_start) {
previous(it, line.rend().base());
char_count--;
}
if (char_count != 0)
cout << "Line " << line_count << ": Error in iterating with previous - wrong number of characters" << '\n';
// Try utf8::iterator
utf8::iterator<string::iterator> u8it(line_start, line_start, line_end);
if (!utf32_line.empty() && *u8it != utf32_line.at(0))
cout << "Line " << line_count << ": Error in utf::iterator * operator" << '\n';
if (std::distance(u8it, utf8::iterator<string::iterator>(line_end, line_start, line_end)) != static_cast<int>(utf32_line.size()))
cout << "Line " << line_count << ": Error in using utf::iterator with std::distance - wrong number of characters" << '\n';
std::advance(u8it, utf32_line.size());
if (u8it != utf8::iterator<string::iterator>(line_end, line_start, line_end))
cout << "Line " << line_count << ": Error in using utf::iterator with std::advance" << '\n';
//======================== Now, the unchecked versions ======================
// Convert it to utf-16 and compare to the checked version
vector<unsigned short> utf16_line_unchecked;
unchecked::utf8to16(line_start, line_end, back_inserter(utf16_line_unchecked));
if (utf16_line != utf16_line_unchecked)
cout << "Line " << line_count << ": Error in unchecked::utf8to16" << '\n';
// Back to utf-8 and compare it to the original line.
back_to_utf8.clear();
unchecked::utf16to8(utf16_line_unchecked.begin(), utf16_line_unchecked.end(), back_inserter(back_to_utf8));
if (back_to_utf8.compare(string(line_start, line_end)) != 0)
cout << "Line " << line_count << ": Unchecked conversion to UTF-16 and back failed" << '\n';
// Now, convert it to utf-32, back to utf-8 and compare
vector <unsigned> utf32_line_unchecked;
unchecked::utf8to32(line_start, line_end, back_inserter(utf32_line_unchecked));
if (utf32_line != utf32_line_unchecked)
cout << "Line " << line_count << ": Error in unchecked::utf8to32" << '\n';
back_to_utf8.clear();
unchecked::utf32to8(utf32_line.begin(), utf32_line.end(), back_inserter(back_to_utf8));
if (back_to_utf8.compare(string(line_start, line_end)) != 0)
cout << "Line " << line_count << ": Unchecked conversion to UTF-32 and back failed" << '\n';
// Now, iterate and back
char_count = 0;
it = line_start;
while (it != line_end) {
unsigned int next_cp = unchecked::peek_next(it);
if (unchecked::next(it) != next_cp)
cout << "Line " << line_count << ": Error: unchecked::peek_next gave a different result than unchecked::next" << '\n';;
char_count++;
}
if (char_count != utf32_line.size())
cout << "Line " << line_count << ": Error in iterating with unchecked::next - wrong number of characters" << '\n';
adv_it = line_start;
utf8::unchecked::advance(adv_it, char_count);
if (adv_it != line_end)
cout << "Line " << line_count << ": Error in unchecked::advance function" << '\n';
if (string::size_type(utf8::unchecked::distance(line_start, line_end)) != char_count)
cout << "Line " << line_count << ": Error in unchecked::distance function" << '\n';
while (it != line_start) {
unchecked::previous(it);
char_count--;
}
if (char_count != 0)
cout << "Line " << line_count << ": Error in iterating with unchecked::previous - wrong number of characters" << '\n';
// Try utf8::unchecked::iterator
utf8::unchecked::iterator<string::iterator> un_u8it(line_start);
if (!utf32_line.empty() && *un_u8it != utf32_line.at(0))
cout << "Line " << line_count << ": Error in utf::unchecked::iterator * operator" << '\n';
if (std::distance(un_u8it, utf8::unchecked::iterator<string::iterator>(line_end)) != static_cast<int>(utf32_line.size()))
cout << "Line " << line_count << ": Error in using utf::unchecked::iterator with std::distance - wrong number of characters" << '\n';
std::advance(un_u8it, utf32_line.size());
if (un_u8it != utf8::unchecked::iterator<string::iterator>(line_end))
cout << "Line " << line_count << ": Error in using utf::unchecked::iterator with std::advance" << '\n';
}
}

2
src/core/Phone.h
View File

@ -1,4 +1,4 @@
#pragma once
#pragma once
#include "EnumConverter.h"

15
src/exporters/JsonExporter.cpp
View File

@ -1,11 +1,17 @@
#include "JsonExporter.h"
#include "exporterTools.h"
#include <utf8.h>
using std::string;
string escapeJsonString(const string& s) {
// JavaScript uses UTF-16 internally. As a result, character escaping in JSON strings is UTF-16-based.
// Convert string to UTF-16
std::u16string utf16String;
utf8::utf8to16(s.begin(), s.end(), std::back_inserter(utf16String));
string result;
for (char c : s) {
for (char16_t c : utf16String) {
switch (c) {
case '"': result += "\\\""; break;
case '\\': result += "\\\\"; break;
@ -15,10 +21,13 @@ string escapeJsonString(const string& s) {
case '\r': result += "\\r"; break;
case '\t': result += "\\t"; break;
default:
if (c <= '\x1f') {
{
bool needsEscaping = c < '\x20' || c >= 0x80;
if (needsEscaping) {
result += fmt::format("\\u{0:04x}", c);
} else {
result += c;
result += static_cast<char>(c);
}
}
}
}

6
src/lib/rhubarbLib.cpp
View File

@ -6,13 +6,13 @@
#include "WaveFileReader.h"
using boost::optional;
using std::u32string;
using std::string;
using boost::filesystem::path;
using std::unique_ptr;
JoiningContinuousTimeline<Shape> animateAudioClip(
const AudioClip& audioClip,
optional<u32string> dialog,
optional<string> dialog,
const ShapeSet& targetShapeSet,
int maxThreadCount,
ProgressSink& progressSink)
@ -32,7 +32,7 @@ unique_ptr<AudioClip> createWaveAudioClip(path filePath) {
JoiningContinuousTimeline<Shape> animateWaveFile(
path filePath,
optional<u32string> dialog,
optional<string> dialog,
const ShapeSet& targetShapeSet,
int maxThreadCount,
ProgressSink& progressSink)

4
src/lib/rhubarbLib.h
View File

@ -9,14 +9,14 @@
JoiningContinuousTimeline<Shape> animateAudioClip(
const AudioClip& audioClip,
boost::optional<std::u32string> dialog,
boost::optional<std::string> dialog,
const ShapeSet& targetShapeSet,
int maxThreadCount,
ProgressSink& progressSink);
JoiningContinuousTimeline<Shape> animateWaveFile(
boost::filesystem::path filePath,
boost::optional<std::u32string> dialog,
boost::optional<std::string> dialog,
const ShapeSet& targetShapeSet,
int maxThreadCount,
ProgressSink& progressSink);

22
src/main.cpp
View File

@ -24,10 +24,11 @@
#include <boost/iostreams/device/null.hpp>
#include "targetShapeSet.h"
#include <boost/utility/in_place_factory.hpp>
#include "platformTools.h"
using std::exception;
using std::string;
using std::u32string;
using std::string;
using std::vector;
using std::unique_ptr;
using std::make_unique;
@ -97,7 +98,14 @@ ShapeSet getTargetShapeSet(const string& extendedShapesString) {
return result;
}
int main(int argc, char *argv[]) {
int main(int platformArgc, char *platformArgv[]) {
// Use UTF-8 throughout
useUtf8ForConsole();
useUtf8ForBoostFilesystem();
// Convert command-line arguments to UTF-8
const vector<string> args = argsToUtf8(platformArgc, platformArgv);
auto pausableStderrSink = addPausableStdErrSink(logging::Level::Warn);
pausableStderrSink->pause();
@ -130,7 +138,11 @@ int main(int argc, char *argv[]) {
});
// Parse command line
cmd.parse(argc, argv);
{
// TCLAP mutates the function argument! Pass a copy.
vector<string> argsCopy(args);
cmd.parse(argsCopy);
}
if (quietMode.getValue()) {
infoStream = &nullStream;
}
@ -146,7 +158,7 @@ int main(int argc, char *argv[]) {
}
logging::infoFormat("Application startup. Command line: {}", join(
vector<char*>(argv, argv + argc) | transformed([](char* arg) { return fmt::format("\"{}\"", arg); }), " "));
args | transformed([](string arg) { return fmt::format("\"{}\"", arg); }), " "));
try {
*infoStream << fmt::format("Generating lip sync data for {}.", inputFilePath) << std::endl;
@ -158,7 +170,7 @@ int main(int argc, char *argv[]) {
// Animate the recording
animation = animateWaveFile(
inputFilePath,
dialogFile.isSet() ? readUtf8File(path(dialogFile.getValue())) : boost::optional<u32string>(),
dialogFile.isSet() ? readUtf8File(path(dialogFile.getValue())) : boost::optional<string>(),
targetShapeSet,
maxThreadCount.getValue(),
progressBar);

2
src/recognition/g2p.cpp
View File

@ -1,4 +1,4 @@
#include <g2p.h>
#include <g2p.h>
#include <regex>
#include "stringTools.h"
#include "logging.h"

1
src/recognition/languageModels.cpp
View File

@ -11,7 +11,6 @@
#include <gsl_util.h>
using std::string;
using std::u32string;
using std::vector;
using std::regex;
using std::map;

10
src/recognition/phoneRecognition.cpp
View File

@ -44,7 +44,7 @@ using std::regex;
using std::regex_replace;
using std::chrono::duration;
using boost::optional;
using std::u32string;
using std::string;
using std::chrono::duration_cast;
using std::array;
@ -251,7 +251,7 @@ lambda_unique_ptr<ngram_model_t> createDefaultLanguageModel(ps_decoder_t& decode
return std::move(result);
}
lambda_unique_ptr<ngram_model_t> createDialogLanguageModel(ps_decoder_t& decoder, const u32string& dialog) {
lambda_unique_ptr<ngram_model_t> createDialogLanguageModel(ps_decoder_t& decoder, const string& dialog) {
// Split dialog into normalized words
vector<string> words = tokenizeText(dialog, [&](const string& word) { return dictionaryContains(*decoder.dict, word); });
@ -264,7 +264,7 @@ lambda_unique_ptr<ngram_model_t> createDialogLanguageModel(ps_decoder_t& decoder
return createLanguageModel(words, decoder);
}
lambda_unique_ptr<ngram_model_t> createBiasedLanguageModel(ps_decoder_t& decoder, const u32string& dialog) {
lambda_unique_ptr<ngram_model_t> createBiasedLanguageModel(ps_decoder_t& decoder, const string& dialog) {
auto defaultLanguageModel = createDefaultLanguageModel(decoder);
auto dialogLanguageModel = createDialogLanguageModel(decoder, dialog);
constexpr int modelCount = 2;
@ -281,7 +281,7 @@ lambda_unique_ptr<ngram_model_t> createBiasedLanguageModel(ps_decoder_t& decoder
return std::move(result);
}
lambda_unique_ptr<ps_decoder_t> createDecoder(optional<u32string> dialog) {
lambda_unique_ptr<ps_decoder_t> createDecoder(optional<string> dialog) {
lambda_unique_ptr<cmd_ln_t> config(
cmd_ln_init(
nullptr, ps_args(), true,
@ -435,7 +435,7 @@ Timeline<Phone> utteranceToPhones(
BoundedTimeline<Phone> recognizePhones(
const AudioClip& inputAudioClip,
optional<u32string> dialog,
optional<string> dialog,
int maxThreadCount,
ProgressSink& progressSink)
{

2
src/recognition/phoneRecognition.h
View File

@ -7,6 +7,6 @@
BoundedTimeline<Phone> recognizePhones(
const AudioClip& audioClip,
boost::optional<std::u32string> dialog,
boost::optional<std::string> dialog,
int maxThreadCount,
ProgressSink& progressSink);

10
src/recognition/tokenization.cpp
View File

@ -10,7 +10,6 @@ extern "C" {
}
using std::runtime_error;
using std::u32string;
using std::string;
using std::vector;
using std::regex;
@ -34,9 +33,12 @@ static const cst_synth_module synth_method_normalize[] = {
};
vector<string> tokenizeViaFlite(const string& text) {
// Convert text to ASCII
const string asciiText = utf8ToAscii(text);
// Create utterance object with text
lambda_unique_ptr<cst_utterance> utterance(new_utterance(), [](cst_utterance* utterance) { delete_utterance(utterance); });
utt_set_input_text(utterance.get(), text.c_str());
utt_set_input_text(utterance.get(), asciiText.c_str());
lambda_unique_ptr<cst_voice> voice = createDummyVoice();
utt_init(utterance.get(), voice.get());
@ -73,8 +75,8 @@ optional<string> findSimilarDictionaryWord(const string& word, function<bool(con
return boost::none;
}
vector<string> tokenizeText(const u32string& text, function<bool(const string&)> dictionaryContains) {
vector<string> words = tokenizeViaFlite(toAscii(text));
vector<string> tokenizeText(const string& text, function<bool(const string&)> dictionaryContains) {
vector<string> words = tokenizeViaFlite(text);
// Join words separated by apostophes
for (int i = words.size() - 1; i > 0; --i) {

2
src/recognition/tokenization.h
View File

@ -4,4 +4,4 @@
#include <functional>
#include <string>
std::vector<std::string> tokenizeText(const std::u32string& text, std::function<bool(const std::string&)> dictionaryContains);
std::vector<std::string> tokenizeText(const std::string& text, std::function<bool(const std::string&)> dictionaryContains);

94
src/tools/asciiCases.cpp
View File

@ -1,94 +0,0 @@
// Generated by asciiCases.rb; don't modify by hand!
case U'À': case U'Á': case U'Â': case U'Ã': case U'Ä': case U'Å': case U'Ā': case U'Ă': case U'Ą': case U'Ǎ': case U'Ǟ': case U'Ǡ': case U'Ǻ': case U'Ȁ': case U'Ȃ': case U'Ȧ': case U'Ⱥ':
return 'A';
case U'Ɓ': case U'Ƃ': case U'Ƀ':
return 'B';
case U'Ç': case U'Ć': case U'Ĉ': case U'Ċ': case U'Č': case U'Ƈ': case U'Ȼ':
return 'C';
case U'Ď': case U'Đ': case U'Ɗ': case U'Ƌ':
return 'D';
case U'È': case U'É': case U'Ê': case U'Ë': case U'Ē': case U'Ĕ': case U'Ė': case U'Ę': case U'Ě': case U'Ȅ': case U'Ȇ': case U'Ȩ': case U'Ɇ':
return 'E';
case U'Ƒ':
return 'F';
case U'Ĝ': case U'Ğ': case U'Ġ': case U'Ģ': case U'Ɠ': case U'Ǥ': case U'Ǧ': case U'Ǵ':
return 'G';
case U'Ĥ': case U'Ħ': case U'Ȟ':
return 'H';
case U'Ì': case U'Í': case U'Î': case U'Ï': case U'Ĩ': case U'Ī': case U'Ĭ': case U'Į': case U'İ': case U'Ɨ': case U'Ǐ': case U'Ȉ': case U'Ȋ':
return 'I';
case U'Ĵ': case U'Ɉ':
return 'J';
case U'Ķ': case U'Ƙ': case U'Ǩ':
return 'K';
case U'Ĺ': case U'Ļ': case U'Ľ': case U'Ŀ': case U'Ł': case U'Ƚ':
return 'L';
case U'Ñ': case U'Ń': case U'Ņ': case U'Ň': case U'Ɲ': case U'Ǹ': case U'Ƞ':
return 'N';
case U'Ò': case U'Ó': case U'Ô': case U'Õ': case U'Ö': case U'Ø': case U'Ō': case U'Ŏ': case U'Ő': case U'Ɵ': case U'Ơ': case U'Ǒ': case U'Ǫ': case U'Ǭ': case U'Ǿ': case U'Ȍ': case U'Ȏ': case U'Ȫ': case U'Ȭ': case U'Ȯ': case U'Ȱ':
return 'O';
case U'Ƥ':
return 'P';
case U'Ŕ': case U'Ŗ': case U'Ř': case U'Ȑ': case U'Ȓ': case U'Ɍ':
return 'R';
case U'Ś': case U'Ŝ': case U'Ş': case U'Š': case U'Ș':
return 'S';
case U'Ţ': case U'Ť': case U'Ŧ': case U'Ƭ': case U'Ʈ': case U'Ț': case U'Ⱦ':
return 'T';
case U'Ù': case U'Ú': case U'Û': case U'Ü': case U'Ũ': case U'Ū': case U'Ŭ': case U'Ů': case U'Ű': case U'Ų': case U'Ư': case U'Ǔ': case U'Ǖ': case U'Ǘ': case U'Ǚ': case U'Ǜ': case U'Ȕ': case U'Ȗ': case U'Ʉ':
return 'U';
case U'Ʋ':
return 'V';
case U'Ŵ':
return 'W';
case U'Ý': case U'Ŷ': case U'Ÿ': case U'Ƴ': case U'Ȳ': case U'Ɏ':
return 'Y';
case U'Ź': case U'Ż': case U'Ž': case U'Ƶ': case U'Ȥ':
return 'Z';
case U'à': case U'á': case U'â': case U'ã': case U'ä': case U'å': case U'ā': case U'ă': case U'ą': case U'ǎ': case U'ǟ': case U'ǡ': case U'ǻ': case U'ȁ': case U'ȃ': case U'ȧ':
return 'a';
case U'ƀ': case U'ƃ':
return 'b';
case U'ç': case U'ć': case U'ĉ': case U'ċ': case U'č': case U'ƈ': case U'ȼ':
return 'c';
case U'ď': case U'đ': case U'ƌ': case U'ȡ':
return 'd';
case U'è': case U'é': case U'ê': case U'ë': case U'ē': case U'ĕ': case U'ė': case U'ę': case U'ě': case U'ȅ': case U'ȇ': case U'ȩ': case U'ɇ':
return 'e';
case U'ƒ':
return 'f';
case U'ĝ': case U'ğ': case U'ġ': case U'ģ': case U'ǥ': case U'ǧ': case U'ǵ':
return 'g';
case U'ĥ': case U'ħ': case U'ȟ':
return 'h';
case U'ì': case U'í': case U'î': case U'ï': case U'ĩ': case U'ī': case U'ĭ': case U'į': case U'ǐ': case U'ȉ': case U'ȋ':
return 'i';
case U'ĵ': case U'ǰ': case U'ɉ':
return 'j';
case U'ķ': case U'ƙ': case U'ǩ':
return 'k';
case U'ĺ': case U'ļ': case U'ľ': case U'ŀ': case U'ł': case U'ƚ': case U'ȴ':
return 'l';
case U'ñ': case U'ń': case U'ņ': case U'ň': case U'ʼn': case U'ƞ': case U'ǹ': case U'ȵ':
return 'n';
case U'ò': case U'ó': case U'ô': case U'õ': case U'ö': case U'ø': case U'ō': case U'ŏ': case U'ő': case U'ơ': case U'ǒ': case U'ǫ': case U'ǭ': case U'ǿ': case U'ȍ': case U'ȏ': case U'ȫ': case U'ȭ': case U'ȯ': case U'ȱ':
return 'o';
case U'ƥ':
return 'p';
case U'ɋ':
return 'q';
case U'ŕ': case U'ŗ': case U'ř': case U'ȑ': case U'ȓ': case U'ɍ':
return 'r';
case U'ś': case U'ŝ': case U'ş': case U'š': case U'ș': case U'ȿ':
return 's';
case U'ţ': case U'ť': case U'ŧ': case U'ƫ': case U'ƭ': case U'ț': case U'ȶ':
return 't';
case U'ù': case U'ú': case U'û': case U'ü': case U'ũ': case U'ū': case U'ŭ': case U'ů': case U'ű': case U'ų': case U'ư': case U'ǔ': case U'ǖ': case U'ǘ': case U'ǚ': case U'ǜ': case U'ȕ': case U'ȗ':
return 'u';
case U'ŵ':
return 'w';
case U'ý': case U'ÿ': case U'ŷ': case U'ƴ': case U'ȳ': case U'ɏ':
return 'y';
case U'ź': case U'ż': case U'ž': case U'ƶ': case U'ȥ': case U'ɀ':
return 'z';

33
src/tools/asciiCases.rb
View File

@ -1,33 +0,0 @@
require 'open-uri'
require 'csv'
# Create mapping from ASCII characters to related Unicode characters
mapping = Hash.new{ |hash, key| hash[key] = [] }
url = 'http://www.unicode.org/Public/8.0.0/ucd/UnicodeData.txt'
headers = [:code, :name, :category]
CSV.new(open(url), :col_sep => ';', :headers => headers).each do |row|
code = row[:code].hex
next if code < 0x80
break if code > 0x24f
char = [code].pack('U')
name = row[:name]
match = /^LATIN (CAPITAL|SMALL) LETTER ([A-Z])\b(?!.*\bLETTER\b)/.match(name)
if match
baseChar = match[2]
if match[1] == 'SMALL'
baseChar = (baseChar.ord + 0x20).chr
end
mapping[baseChar] << char
end
end
mapping = mapping.sort.to_h
# Generate asciiCases.cpp
File.open('asciiCases.cpp', 'w') do |file|
file.print "// Generated by #{__FILE__}; don't modify by hand!\n\n"
mapping.each do |asciiChar, unicodeChars|
file.print unicodeChars.map { |c| "case U'#{c}':" }.join(' '), "\n"
file.print "\treturn '#{asciiChar}';\n"
end
end

82
src/tools/platformTools.cpp
View File

@ -6,9 +6,21 @@
#include <boost/uuid/uuid_io.hpp>
#include "platformTools.h"
#include <whereami.h>
#include <utf8.h>
#include <gsl_util.h>
#include "tools.h"
#include <boost/filesystem/detail/utf8_codecvt_facet.hpp>
#include <iostream>
#ifdef _WIN32
#include <Windows.h>
#include <io.h>
#include <fcntl.h>
#endif
using boost::filesystem::path;
using std::string;
using std::vector;
path getBinPath() {
static const path binPath = [] {
@ -69,3 +81,73 @@ std::string errorNumberToString(int errorNumber) {
#endif
return message;
}
vector<string> argsToUtf8(int argc, char* argv[]) {
#ifdef _WIN32
// On Windows, there is no way to convert the single-byte argument strings to Unicode.
// We'll just ignore them.
UNUSED(argc);
UNUSED(argv);
// Get command-line arguments as UTF16 strings
int argumentCount;
static_assert(sizeof(wchar_t) == sizeof(char16_t), "Expected wchar_t to be a 16-bit type.");
char16_t** args = reinterpret_cast<char16_t**>(CommandLineToArgvW(GetCommandLineW(), &argumentCount));
if (!args) {
throw std::runtime_error("Error splitting the UTF-16 command line arguments.");
}
auto freeArgs = gsl::finally([&]() { LocalFree(args); });
assert(argumentCount == argc);
// Convert UTF16 strings to UTF8
vector<string> result;
for (int i = 0; i < argc; ++i) {
std::u16string utf16String(args[i]);
string utf8String;
utf8::utf16to8(utf16String.begin(), utf16String.end(), back_inserter(utf8String));
result.push_back(utf8String);
}
return result;
#else
// On Unix systems, command-line args are already in UTF-8 format. Just convert them to strings.
vector<string> result;
for (int i = 0; i < argc; ++i) {
result.push_back(string(argv[i]));
}
return result;
#endif
}
class ConsoleBuffer : public std::stringbuf {
public:
explicit ConsoleBuffer(FILE* file)
: file(file) {}
int sync() override {
fputs(str().c_str(), file);
str("");
return 0;
}
private:
FILE* file;
};
void useUtf8ForConsole() {
// Unix systems already expect UTF-8-encoded data
#ifdef _WIN32
// Set console code page to UTF-8 so the console knows how to interpret string data
SetConsoleOutputCP(CP_UTF8);
// Prevent default stream buffer from chopping up UTF-8 byte sequences.
// See https://stackoverflow.com/questions/45575863/how-to-print-utf-8-strings-to-stdcout-on-windows
std::cout.rdbuf(new ConsoleBuffer(stdout));
std::cerr.rdbuf(new ConsoleBuffer(stderr));
#endif
}
void useUtf8ForBoostFilesystem() {
std::locale globalLocale = std::locale();
std::locale utf8Locale(globalLocale, new boost::filesystem::detail::utf8_codecvt_facet);
path::imbue(utf8Locale);
}

5
src/tools/platformTools.h
View File

@ -10,3 +10,8 @@ boost::filesystem::path getTempFilePath();
std::tm getLocalTime(const time_t& time);
std::string errorNumberToString(int errorNumber);
std::vector<std::string> argsToUtf8(int argc, char *argv[]);
void useUtf8ForConsole();
void useUtf8ForBoostFilesystem();

87
src/tools/stringTools.cpp
View File

@ -1,12 +1,16 @@
#include "stringTools.h"
#include "stringTools.h"
#include <boost/algorithm/string/trim.hpp>
#include <codecvt>
#include <utf8.h>
#include <utf8proc.h>
#include <regex>
using std::string;
using std::wstring;
using std::u32string;
using std::vector;
using boost::optional;
using std::regex;
using std::regex_replace;
vector<string> splitIntoLines(const string& s) {
vector<string> lines;
@ -83,6 +87,10 @@ vector<string> wrapString(const string& s, int lineLength, int hangingIndent) {
return lines;
}
bool isValidUtf8(const string& s) {
return utf8::is_valid(s.begin(), s.end());
}
wstring latin1ToWide(const string& s) {
wstring result;
for (unsigned char c : s) {
@ -91,40 +99,61 @@ wstring latin1ToWide(const string& s) {
return result;
}
optional<char> toAscii(char32_t ch) {
switch (ch) {
#include "asciiCases.cpp"
default:
return ch < 0x80 ? static_cast<char>(ch) : optional<char>();
string utf8ToAscii(const string s) {
// Normalize string, simplifying it as much as possible
const NormalizationOptions options = NormalizationOptions::CompatibilityMode
| NormalizationOptions::Decompose
| NormalizationOptions::SimplifyLineBreaks
| NormalizationOptions::SimplifyWhiteSpace
| NormalizationOptions::StripCharacterMarkings
| NormalizationOptions::StripIgnorableCharacters;
string simplified = normalizeUnicode(s, options);
// Replace common Unicode characters with ASCII equivalents
static const vector<std::pair<regex, string>> replacements{
{regex("«|»|“|”|„|‟"), "\""},
{regex("‘|’|‚|‛|‹|›"), "'"},
{regex("‐|‑|‒|⁃|⁻|₋|−|➖|–|—|―|﹘|﹣|－"), "-"},
{regex("…|⋯"), "..."},
{regex("•"), "*"},
{regex("†|＋"), "+"},
{regex("⁄|∕|⧸|／|/"), "/"},
{regex("×"), "x"},
};
for (const auto& replacement : replacements) {
simplified = regex_replace(simplified, replacement.first, replacement.second);
}
// Skip all non-ASCII code points, including multi-byte characters
string result;
for (char c : simplified) {
const bool isAscii = (c & 0x80) == 0;
if (isAscii) {
result.append(1, c);
}
}
string toAscii(const u32string& s) {
string result;
for (char32_t ch : s) {
optional<char> ascii = toAscii(ch);
if (ascii) result.append(1, *ascii);
}
return result;
}
string toAscii(const wstring& s) {
string result;
for (wchar_t ch : s) {
optional<char> ascii = toAscii(ch);
if (ascii) result.append(1, *ascii);
string normalizeUnicode(const string s, NormalizationOptions options) {
char* result;
const utf8proc_ssize_t charCount = utf8proc_map(
reinterpret_cast<const uint8_t*>(s.data()),
s.length(),
reinterpret_cast<uint8_t**>(&result),
static_cast<utf8proc_option_t>(options));
if (charCount < 0) {
const utf8proc_ssize_t errorCode = charCount;
const string message = string("Error normalizing string: ") + utf8proc_errmsg(errorCode);
if (errorCode == UTF8PROC_ERROR_INVALIDOPTS) {
throw std::invalid_argument(message);
}
return result;
throw std::runtime_error(message);
}
u32string utf8ToUtf32(const string& s) {
#if defined(_MSC_VER) && _MSC_VER <= 1900
// Workaround for Visual Studio 2015
// See https://connect.microsoft.com/VisualStudio/feedback/details/1403302/unresolved-external-when-using-codecvt-utf8
std::wstring_convert<std::codecvt_utf8<uint32_t>, uint32_t> convert;
return u32string(reinterpret_cast<const char32_t*>(convert.from_bytes(s).c_str()));
#else
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> convert;
return convert.from_bytes(s);
#endif
string resultString(result, charCount);
free(result);
return resultString;
}

23
src/tools/stringTools.h
View File

@ -3,6 +3,7 @@
#include <vector>
#include <boost/optional.hpp>
#include <boost/lexical_cast.hpp>
#include <utf8proc.h>
std::vector<std::string> splitIntoLines(const std::string& s);
@ -10,15 +11,31 @@ std::vector<std::string> wrapSingleLineString(const std::string& s, int lineLeng
std::vector<std::string> wrapString(const std::string& s, int lineLength, int hangingIndent = 0);
bool isValidUtf8(const std::string& s);
std::wstring latin1ToWide(const std::string& s);
boost::optional<char> toAscii(char32_t ch);
std::string toAscii(const std::u32string& s);
std::string utf8ToAscii(const std::string s);
std::string toAscii(const std::wstring& s);
enum class NormalizationOptions : int {
CompatibilityMode = UTF8PROC_COMPAT,
Compose = UTF8PROC_COMPOSE,
Decompose = UTF8PROC_DECOMPOSE,
StripIgnorableCharacters = UTF8PROC_IGNORE,
ThrowOnUnassignedCodepoints = UTF8PROC_REJECTNA,
SimplifyLineBreaks = UTF8PROC_NLF2LS,
SimplifyWhiteSpace = UTF8PROC_STRIPCC,
StripCharacterMarkings = UTF8PROC_STRIPMARK
};
std::u32string utf8ToUtf32(const std::string& s);
constexpr NormalizationOptions
operator|(NormalizationOptions a, NormalizationOptions b) {
return static_cast<NormalizationOptions>(static_cast<int>(a) | static_cast<int>(b));
}
std::string normalizeUnicode(const std::string s, NormalizationOptions options);
template<typename T>
std::string join(T range, const std::string separator) {

13
src/tools/textFiles.cpp
View File

@ -5,10 +5,9 @@
#include "stringTools.h"
using std::string;
using std::u32string;
using boost::filesystem::path;
u32string readUtf8File(path filePath) {
string readUtf8File(path filePath) {
if (!exists(filePath)) {
throw std::invalid_argument(fmt::format("File {} does not exist.", filePath));
}
@ -16,12 +15,12 @@ u32string readUtf8File(path filePath) {
boost::filesystem::ifstream file;
file.exceptions(std::ifstream::failbit | std::ifstream::badbit);
file.open(filePath);
string utf8Text((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
try {
return utf8ToUtf32(utf8Text);
} catch (...) {
std::throw_with_nested(std::runtime_error(fmt::format("File encoding is not ASCII or UTF-8.", filePath)));
string text((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
if (!isValidUtf8(text)) {
throw std::runtime_error("File encoding is not ASCII or UTF-8.");
}
return text;
} catch (...) {
std::throw_with_nested(std::runtime_error(fmt::format("Error reading file {0}.", filePath)));
}

2
src/tools/textFiles.h
View File

@ -2,4 +2,4 @@
#include <boost/filesystem/path.hpp>
std::u32string readUtf8File(boost::filesystem::path filePath);
std::string readUtf8File(boost::filesystem::path filePath);

12
tests/stringToolsTests.cpp
View File

@ -81,11 +81,15 @@ TEST(latin1ToWide, basic) {
EXPECT_EQ(pangramWide, latin1ToWide(pangramLatin1));
}
// toAscii
// utf8ToAscii
TEST(toAscii, string) {
TEST(utf8ToAscii, string) {
EXPECT_EQ(
"A naive man called was having pina colada and creme brulee.",
toAscii(U"A naïve man called 晨 was having piña colada and crème brûlée."));
EXPECT_EQ(string(""), toAscii(U""));
utf8ToAscii("A naïve man called 晨 was having piña colada and crème brûlée."));
EXPECT_EQ(string(""), utf8ToAscii(""));
EXPECT_EQ(string("- - - - - - - - - -"), utf8ToAscii("- ‐ ‑ ‒ – — ― ﹘ ﹣ －"));
EXPECT_EQ(string("' ' ' ' \" \" \" \" \" \""), utf8ToAscii("‘ ’ ‚ ‛ “ ” „ ‟ « »"));
EXPECT_EQ(string("1 2 3"), utf8ToAscii("¹ ² ³"));
EXPECT_EQ(string("1/4 1/2 3/4"), utf8ToAscii("¼ ½ ¾"));
}

32
tests/tokenizationTests.cpp
View File

@ -2,6 +2,7 @@
#include "tokenization.h"
#include <regex>
#include <unordered_set>
#include <utf8.h>
using namespace testing;
using std::string;
@ -14,57 +15,64 @@ bool returnTrue(const string&) {
}
TEST(tokenizeText, simpleCases) {
EXPECT_THAT(tokenizeText(U"", returnTrue), IsEmpty());
EXPECT_THAT(tokenizeText(U" \t\n\r\n ", returnTrue), IsEmpty());
EXPECT_THAT(tokenizeText("", returnTrue), IsEmpty());
EXPECT_THAT(tokenizeText(" \t\n\r\n ", returnTrue), IsEmpty());
EXPECT_THAT(
tokenizeText(U"Wit is educated insolence.", returnTrue),
tokenizeText("Wit is educated insolence.", returnTrue),
ElementsAre("wit", "is", "educated", "insolence")
);
}
TEST(tokenizeText, numbers) {
EXPECT_THAT(
tokenizeText(U"Henry V died at 36.", returnTrue),
tokenizeText("Henry V died at 36.", returnTrue),
ElementsAre("henry", "the", "fifth", "died", "at", "thirty", "six")
);
EXPECT_THAT(
tokenizeText(U"I spent $4.50 on gum.", returnTrue),
tokenizeText("I spent $4.50 on gum.", returnTrue),
ElementsAre("i", "spent", "four", "dollars", "fifty", "cents", "on", "gum")
);
EXPECT_THAT(
tokenizeText(U"I was born in 1982.", returnTrue),
tokenizeText("I was born in 1982.", returnTrue),
ElementsAre("i", "was", "born", "in", "nineteen", "eighty", "two")
);
}
TEST(tokenizeText, abbreviations) {
EXPECT_THAT(
tokenizeText(U"Prof. Foo lives on Dr. Dolittle Dr.", [](const string& word) { return word == "prof."; }),
tokenizeText("Prof. Foo lives on Dr. Dolittle Dr.", [](const string& word) { return word == "prof."; }),
ElementsAre("prof.", "foo", "lives", "on", "doctor", "dolittle", "drive")
);
}
TEST(tokenizeText, apostrophes) {
EXPECT_THAT(
tokenizeText(U"'Tis said he'd wish'd for a 'bus 'cause he wouldn't walk.", [](const string& word) { return word == "wouldn't"; }),
tokenizeText("'Tis said he'd wish'd for a 'bus 'cause he wouldn't walk.", [](const string& word) { return word == "wouldn't"; }),
ElementsAreArray(vector<string>{ "tis", "said", "he'd", "wish'd", "for", "a", "bus", "cause", "he", "wouldn't", "walk" })
);
}
TEST(tokenizeText, math) {
EXPECT_THAT(
tokenizeText(U"'1+2*3=7", returnTrue),
tokenizeText("'1+2*3=7", returnTrue),
ElementsAre("one", "plus", "two", "times", "three", "equals", "seven")
);
}
TEST(tokenizeText, unicodeCharacters) {
EXPECT_THAT(
tokenizeText("A naïve man called 晨 had piña colada and crème brûlée.", returnTrue),
ElementsAre("a", "naive", "man", "called", "had", "pina", "colada", "and", "creme", "brulee")
);
}
// Checks that each word contains only the characters a-z and the apostrophe
TEST(tokenizeText, wordsUseLimitedCharacters) {
// Create string containing lots of undesirable characters
u32string input = U"A naïve man called 晨 was having piña colada and crème brûlée.";
string input = "A naïve man called 晨 was having piña colada and crème brûlée.";
for (char32_t c = 0; c <= 1000; ++c) {
input.append(U" ");
input.append(1, c);
input.append(" ");
utf8::append(c, back_inserter(input));
}
regex legal("^[a-z']+$");