Added utf8proc library

2017-08-01 19:10:03 +02:00 · 2017-08-01 19:10:03 +02:00 · 7789d43e47
parent e5c7215cc5
commit 7789d43e47
37 changed files with 19780 additions and 0 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -228,6 +228,15 @@ target_include_directories(utfcpp SYSTEM PUBLIC "lib/utfcpp-2.3.5/source")
 target_compile_options(utfcpp PRIVATE ${disableWarningsFlags})
 set_target_properties(utfcpp PROPERTIES FOLDER lib)

+# ... utf8proc
+add_library(utf8proc
+	lib/utf8proc-2a2f97e1/utf8proc.c
+	lib/utf8proc-2a2f97e1/utf8proc.h
+)
+target_include_directories(utf8proc SYSTEM PUBLIC "lib/utf8proc-2a2f97e1")
+target_compile_options(utf8proc PRIVATE ${disableWarningsFlags})
+set_target_properties(utf8proc PROPERTIES FOLDER lib)
+
 # Define Rhubarb libraries

 # ... rhubarb-animation
--- a/LICENSE.md
+++ b/LICENSE.md
@ -174,6 +174,48 @@ The [UTF8-CPP](https://github.com/nemtrif/utfcpp) library is released under the
 >
 > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

+### `[utf8proc]` utf8proc
+
+The [utf8proc](https://github.com/JuliaLang/utf8proc) library is released under the **MIT License (MIT)**, while some of its data is released under the **UNICODE License**.
+
+> #### utf8proc license
+> 
+> **utf8proc** is a software package originally developed by Jan Behrens and the rest of the Public Software Group, who deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers. Like the original utf8proc, whose copyright and license statements are reproduced below, all new work on the utf8proc library is licensed under the [MIT "expat" license](http://opensource.org/licenses/MIT):
+> 
+> *Copyright &copy; 2014-2015 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.*
+> 
+> Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+> 
+> The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+> 
+> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+> 
+> #### Original utf8proc license
+> 
+> *Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany*
+> 
+> Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+> 
+> The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+> 
+> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+> 
+> #### Unicode data license
+> 
+> This software distribution contains derived data from a modified version of the Unicode data files. The following license applies to that data:
+>
+> **COPYRIGHT AND PERMISSION NOTICE**
+> 
+> *Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed under the Terms of Use in http://www.unicode.org/copyright.html.*
+> 
+> Permission is hereby granted, free of charge, to any person obtaining a copy of the Unicode data files and any associated documentation (the "Data Files") or Unicode software and any associated documentation (the "Software") to deal in the Data Files or Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, and/or sell copies of the Data Files or Software, and to permit persons to whom the Data Files or Software are furnished to do so, provided that (a) the above copyright notice(s) and this permission notice appear with all copies of the Data Files or Software, (b) both the above copyright notice(s) and this permission notice appear in associated documentation, and (c) there is clear notice in each modified Data File or in the Software as well as in the documentation associated with the Data File(s) or Software that the data or software has been modified.
+> 
+> THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+> 
+> Except as contained in this notice, the name of a copyright holder shall not be used in advertising or otherwise to promote the sale, use or other dealings in these Data Files or Software without prior written authorization of the copyright holder.
+> 
+> Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be registered in some jurisdictions. All other trademarks and registered trademarks mentioned herein are the property of their respective owners.
+
 ### `[webrtc]` WebRTC

 The [WebRTC](https://chromium.googlesource.com/external/webrtc) library is released under the **3-clause BSD License**.
--- a/lib/utf8proc-2a2f97e1/.gitignore
+++ b/lib/utf8proc-2a2f97e1/.gitignore
@ -0,0 +1,28 @@
+*.tar.gz
+*.exe
+*.dll
+*.do
+*.o
+*.so*
+*.a
+*.dll
+*.dylib
+*.dSYM
+*.out
+*.new
+data/*.txt
+data/*.ttf
+data/*.sfd
+/docs/
+bench/bench
+bench/icu
+bench/unistring
+test/normtest
+test/graphemetest
+test/printproperty
+test/charwidth
+test/valid
+test/iterate
+test/case
+test/custom
+/tmp/
--- a/lib/utf8proc-2a2f97e1/.travis.yml
+++ b/lib/utf8proc-2a2f97e1/.travis.yml
@ -0,0 +1,22 @@
+language: c
+compiler:
+    - gcc
+    - clang
+notifications:
+    email: false
+before_install:
+    - sudo add-apt-repository ppa:staticfloat/julia-deps -y
+    - sudo add-apt-repository ppa:staticfloat/juliareleases -y
+    - sudo apt-get update -qq -y
+    - sudo apt-get install libpcre3-dev julia fontforge -y
+script:
+    - make manifest && diff MANIFEST.new MANIFEST
+    - make check
+    - make data && diff data/utf8proc_data.c.new utf8proc_data.c
+    - make clean && git status --ignored --porcelain && test -z "$(git status --ignored --porcelain)"
+    - (mkdir build_static && cd build_static && cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON && make)
+    - (mkdir build_shared && cd build_shared && cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DBUILD_SHARED_LIBS=ON && make)
+env:
+    # use JuliaLang caching (https://github.com/staticfloat/cache.julialang.org)
+    # so that Travis builds do not depend on anyone's flaky servers but our own
+    - URLCACHE=https://cache.julialang.org/ CFLAGS="-O2 -Werror -Wmissing-prototypes"
--- a/lib/utf8proc-2a2f97e1/CMakeLists.txt
+++ b/lib/utf8proc-2a2f97e1/CMakeLists.txt
@ -0,0 +1,33 @@
+cmake_minimum_required (VERSION 2.8)
+
+include (utils.cmake)
+
+disallow_intree_builds()
+
+project (utf8proc C)
+
+# This is the ABI version number, which may differ from the
+# API version number (defined in utf8proc.h).
+# Be sure to also update these in Makefile and MANIFEST!
+set(SO_MAJOR 2)
+set(SO_MINOR 1)
+set(SO_PATCH 0)
+
+add_definitions (
+  -DUTF8PROC_EXPORTS
+)
+
+if (NOT MSVC)
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -std=c99 -pedantic -Wall")
+endif ()
+
+add_library (utf8proc
+  utf8proc.c
+  utf8proc.h
+)
+
+set_target_properties (utf8proc PROPERTIES
+  POSITION_INDEPENDENT_CODE ON
+  VERSION "${SO_MAJOR}.${SO_MINOR}.${SO_PATCH}"
+  SOVERSION ${SO_MAJOR}
+)
--- a/lib/utf8proc-2a2f97e1/Doxyfile
+++ b/lib/utf8proc-2a2f97e1/Doxyfile
--- a/lib/utf8proc-2a2f97e1/LICENSE.md
+++ b/lib/utf8proc-2a2f97e1/LICENSE.md
@ -0,0 +1,93 @@
+## utf8proc license ##
+
+**utf8proc** is a software package originally developed
+by Jan Behrens and the rest of the Public Software Group, who
+deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers.  Like the original utf8proc,
+whose copyright and license statements are reproduced below, all new
+work on the utf8proc library is licensed under the [MIT "expat"
+license](http://opensource.org/licenses/MIT):
+
+*Copyright &copy; 2014-2015 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.*
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+## Original utf8proc license ##
+
+*Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany*
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+## Unicode data license ##
+
+This software distribution contains derived data from a modified version of
+the Unicode data files. The following license applies to that data:
+
+**COPYRIGHT AND PERMISSION NOTICE**
+
+*Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
+under the Terms of Use in http://www.unicode.org/copyright.html.*
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of the Unicode data files and any associated documentation (the "Data
+Files") or Unicode software and any associated documentation (the
+"Software") to deal in the Data Files or Software without restriction,
+including without limitation the rights to use, copy, modify, merge,
+publish, distribute, and/or sell copies of the Data Files or Software, and
+to permit persons to whom the Data Files or Software are furnished to do
+so, provided that (a) the above copyright notice(s) and this permission
+notice appear with all copies of the Data Files or Software, (b) both the
+above copyright notice(s) and this permission notice appear in associated
+documentation, and (c) there is clear notice in each modified Data File or
+in the Software as well as in the documentation associated with the Data
+File(s) or Software that the data or software has been modified.
+
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
+INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR
+CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
+USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder shall
+not be used in advertising or otherwise to promote the sale, use or other
+dealings in these Data Files or Software without prior written
+authorization of the copyright holder.
+
+Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be
+registered in some jurisdictions. All other trademarks and registered
+trademarks mentioned herein are the property of their respective owners.
--- a/lib/utf8proc-2a2f97e1/MANIFEST
+++ b/lib/utf8proc-2a2f97e1/MANIFEST
@ -0,0 +1,7 @@
+include/
+include/utf8proc.h
+lib/
+lib/libutf8proc.a
+lib/libutf8proc.so -> libutf8proc.so.2.1.0
+lib/libutf8proc.so.2 -> libutf8proc.so.2.1.0
+lib/libutf8proc.so.2.1.0
--- a/lib/utf8proc-2a2f97e1/Makefile
+++ b/lib/utf8proc-2a2f97e1/Makefile
@ -0,0 +1,149 @@
+# libutf8proc Makefile
+
+# programs
+AR?=ar
+CC?=gcc
+INSTALL=install
+FIND=find
+
+# compiler settings
+CFLAGS ?= -O2
+PICFLAG = -fPIC
+C99FLAG = -std=c99
+WCFLAGS = -Wall -pedantic
+UCFLAGS = $(CFLAGS) $(PICFLAG) $(C99FLAG) $(WCFLAGS) -DUTF8PROC_EXPORTS
+
+# shared-library version MAJOR.MINOR.PATCH ... this may be *different*
+# from the utf8proc version number because it indicates ABI compatibility,
+# not API compatibility: MAJOR should be incremented whenever *binary*
+# compatibility is broken, even if the API is backward-compatible.
+# The API version number is defined in utf8proc.h.
+# Be sure to also update these ABI versions in MANIFEST and CMakeLists.txt!
+MAJOR=2
+MINOR=1
+PATCH=0
+
+OS := $(shell uname)
+ifeq ($(OS),Darwin) # MacOS X
+  SHLIB_EXT = dylib
+  SHLIB_VERS_EXT = $(MAJOR).dylib
+else # GNU/Linux, at least (Windows should probably use cmake)
+  SHLIB_EXT = so
+  SHLIB_VERS_EXT = so.$(MAJOR).$(MINOR).$(PATCH)
+endif
+
+# installation directories (for 'make install')
+prefix=/usr/local
+libdir=$(prefix)/lib
+includedir=$(prefix)/include
+
+# meta targets
+
+.PHONY: all clean data update manifest install
+
+all: libutf8proc.a libutf8proc.$(SHLIB_EXT)
+
+clean:
+	rm -f utf8proc.o libutf8proc.a libutf8proc.$(SHLIB_VERS_EXT) libutf8proc.$(SHLIB_EXT)
+ifneq ($(OS),Darwin)
+	rm -f libutf8proc.so.$(MAJOR)
+endif
+	rm -f test/tests.o test/normtest test/graphemetest test/printproperty test/charwidth test/valid test/iterate test/case test/custom
+	rm -rf MANIFEST.new tmp
+	$(MAKE) -C bench clean
+	$(MAKE) -C data clean
+
+data: data/utf8proc_data.c.new
+
+update: data/utf8proc_data.c.new
+	cp -f data/utf8proc_data.c.new utf8proc_data.c
+
+manifest: MANIFEST.new
+
+# real targets
+
+data/utf8proc_data.c.new: libutf8proc.$(SHLIB_EXT) data/data_generator.rb data/charwidths.jl
+	$(MAKE) -C data utf8proc_data.c.new
+
+utf8proc.o: utf8proc.h utf8proc.c utf8proc_data.c
+	$(CC) $(UCFLAGS) -c -o utf8proc.o utf8proc.c
+
+libutf8proc.a: utf8proc.o
+	rm -f libutf8proc.a
+	$(AR) rs libutf8proc.a utf8proc.o
+
+libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH): utf8proc.o
+	$(CC) $(LDFLAGS) -shared -o $@ -Wl,-soname -Wl,libutf8proc.so.$(MAJOR) utf8proc.o
+	chmod a-x $@
+
+libutf8proc.so: libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH)
+	ln -f -s libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH) $@
+	ln -f -s libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH) $@.$(MAJOR)
+
+libutf8proc.$(MAJOR).dylib: utf8proc.o
+	$(CC) -dynamiclib -o $@ $^ -install_name $(libdir)/$@ -Wl,-compatibility_version -Wl,$(MAJOR) -Wl,-current_version -Wl,$(MAJOR).$(MINOR).$(PATCH)
+
+libutf8proc.dylib: libutf8proc.$(MAJOR).dylib
+	ln -f -s libutf8proc.$(MAJOR).dylib $@
+
+install: libutf8proc.a libutf8proc.$(SHLIB_EXT) libutf8proc.$(SHLIB_VERS_EXT)
+	mkdir -m 755 -p $(DESTDIR)$(includedir)
+	$(INSTALL) -m 644 utf8proc.h $(DESTDIR)$(includedir)
+	mkdir -m 755 -p $(DESTDIR)$(libdir)
+	$(INSTALL) -m 644 libutf8proc.a $(DESTDIR)$(libdir)
+	$(INSTALL) -m 755 libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)
+	ln -f -s libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)/libutf8proc.$(SHLIB_EXT)
+ifneq ($(OS),Darwin)
+	ln -f -s libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)/libutf8proc.so.$(MAJOR)
+endif
+
+MANIFEST.new:
+	rm -rf tmp
+	$(MAKE) install prefix=/usr DESTDIR=$(PWD)/tmp
+	$(FIND) tmp/usr -mindepth 1 -type l -printf "%P -> %l\n" -or -type f -printf "%P\n" -or -type d -printf "%P/\n" | LC_ALL=C sort > $@
+	rm -rf tmp
+
+# Test programs
+
+data/NormalizationTest.txt:
+	$(MAKE) -C data NormalizationTest.txt
+
+data/GraphemeBreakTest.txt:
+	$(MAKE) -C data GraphemeBreakTest.txt
+
+test/tests.o: test/tests.c test/tests.h utf8proc.h
+	$(CC) $(UCFLAGS) -c -o test/tests.o test/tests.c
+
+test/normtest: test/normtest.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/normtest.c test/tests.o utf8proc.o -o $@
+
+test/graphemetest: test/graphemetest.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/graphemetest.c test/tests.o utf8proc.o -o $@
+
+test/printproperty: test/printproperty.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/printproperty.c test/tests.o utf8proc.o -o $@
+
+test/charwidth: test/charwidth.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/charwidth.c test/tests.o utf8proc.o -o $@
+
+test/valid: test/valid.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/valid.c test/tests.o utf8proc.o -o $@
+
+test/iterate: test/iterate.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/iterate.c test/tests.o utf8proc.o -o $@
+
+test/case: test/case.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/case.c test/tests.o utf8proc.o -o $@
+
+test/custom: test/custom.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/custom.c test/tests.o utf8proc.o -o $@
+
+check: test/normtest data/NormalizationTest.txt test/graphemetest data/GraphemeBreakTest.txt test/printproperty test/case test/custom test/charwidth test/valid test/iterate bench/bench.c bench/util.c bench/util.h utf8proc.o
+	$(MAKE) -C bench
+	test/normtest data/NormalizationTest.txt
+	test/graphemetest data/GraphemeBreakTest.txt
+	test/charwidth
+	test/valid
+	test/iterate
+	test/case
+	test/custom
--- a/lib/utf8proc-2a2f97e1/NEWS.md
+++ b/lib/utf8proc-2a2f97e1/NEWS.md
@ -0,0 +1,303 @@
+# utf8proc release history #
+
+## Version 2.1 ##
+
+2016-12-26:
+
+- New functions `utf8proc_map_custom` and `utf8proc_decompose_custom`
+  to allow user-supplied transformations of codepoints, in conjunction
+  with other transformations ([#89]).
+
+- New function `utf8proc_normalize_utf32` to apply normalizations
+  directly to UTF-32 data (not just UTF-8) ([#88]).
+
+- Fixed stack overflow that could occur due to incorrect definition
+  of `UINT16_MAX` with some compilers ([#84]).
+
+- Fixed conflict with `stdbool.h` in Visual Studio ([#90]).
+
+- Updated font metrics to use Unifont 9.0.04.
+
+## Version 2.0.2 ##
+
+2016-07-27:
+
+- Move `-Wmissing-prototypes` warning flag from `Makefile` to `.travis.yml`
+  since MSVC does not understand this flag and it is occasionally useful to
+  build using MSVC through the `Makefile` ([#79]).
+
+- Use a different variable name for a nested loop in `bench/bench.c`, and
+  declare it in a C89 way rather than inside the `for` to avoid "error:
+  'for' loop initial declarations are only allowed in C99 mode" ([#80]).
+
+## Version 2.0.1 ##
+
+2016-07-13:
+
+- Bug fix in `utf8proc_grapheme_break_stateful` ([#77]).
+
+- Tests now use versioned Unicode files, so they will no longer
+  break when a new version of Unicode is released ([#78]).
+
+## Version 2.0 ##
+
+2016-07-13:
+
+- Updated for Unicode 9.0 ([#70]).
+
+- New `utf8proc_grapheme_break_stateful` to handle the complicated
+  grapheme-breaking rules in Unicode 9.  The old `utf8proc_grapheme_break`
+  is still provided, but may incorrectly identify grapheme breaks
+  in some Unicode-9 sequences.
+
+- Smaller Unicode tables ([#62], [#68]).  This required changes
+  in the `utf8proc_property_t` structure, which breaks backward
+  compatibility if you access this `struct` directly.  The
+  functions in the API remain backward-compatible, however.
+
+- Buffer overrun fix ([#66]).
+
+## Version 1.3.1 ##
+
+2015-11-02:
+
+- Do not export symbol for internal function `unsafe_encode_char()` ([#55]).
+
+- Install relative symbolic links for shared libraries ([#58]).
+
+- Enable and fix compiler warnings ([#55], [#58]).
+
+- Add missing files to `make clean` ([#58]).
+
+## Version 1.3 ##
+
+2015-07-06:
+
+- Updated for Unicode 8.0 ([#45]).
+
+- New `utf8proc_tolower` and `utf8proc_toupper` functions, portable
+  replacements for `towlower` and `towupper` in the C library ([#40]).
+
+- Don't treat Unicode "non-characters" as invalid, and improved
+  validity checking in general ([#35]).
+
+- Prefix all typedefs with `utf8proc_`, e.g. `utf8proc_int32_t`,
+  to avoid collisions with other libraries ([#32]).
+
+- Rename `DLLEXPORT` to `UTF8PROC_DLLEXPORT` to prevent collisions.
+
+- Fix build breakage in the benchmark routines.
+
+- More fine-grained Makefile variables (`PICFLAG` etcetera), so that
+  compilation flags can be selectively overridden, and in particular
+  so that `CFLAGS` can be changed without accidentally eliminating
+  necessary flags like `-fPIC` and `-std=c99` ([#43]).
+
+- Updated character-width tables based on Unifont 8.0.01 ([#51]) and
+  the Unicode 8 character categories ([#47]).
+
+## Version 1.2 ##
+
+2015-03-28:
+
+- Updated for Unicode 7.0 ([#6]).
+
+- New function `utf8proc_grapheme_break(c1,c2)` that returns whether
+  there is a grapheme break between `c1` and `c2` ([#20]).
+
+- New function `utf8proc_charwidth(c)` that returns the number of
+  column-positions that should be required for `c`; essentially a
+  portable replacment for `wcwidth(c)` ([#27]).
+
+- New function `utf8proc_category(c)` that returns the Unicode
+  category of `c` (as one of the constants `UTF8PROC_CATEGORY_xx`).
+  Also, a function `utf8proc_category_string(c)` that returns the Unicode
+  category of `c` as a two-character string.
+
+- `cmake` script `CMakeLists.txt`, in addition to `Makefile`, for
+  easier compilation on Windows ([#28]).
+
+- Various `Makefile` improvements: a `make check` target to perform
+  tests ([#13]), `make install`, a rule to automate updating the Unicode
+  tables, etcetera.
+
+- The shared library is now versioned (e.g. has a soname on GNU/Linux) ([#24]).
+
+- C++/MSVC compatibility ([#17]).
+
+- Most `#defined` constants are now `enums` ([#29]).
+
+- New preprocessor constants `UTF8PROC_VERSION_MAJOR`,
+  `UTF8PROC_VERSION_MINOR`, and `UTF8PROC_VERSION_PATCH` for compile-time
+  detection of the API version.
+
+- Doxygen-formatted documentation ([#29]).
+
+- The Ruby and PostgreSQL plugins have been removed due to lack of testing ([#22]).
+
+## Version 1.1.6 ##
+
+2013-11-27:
+
+- PostgreSQL 9.2 and 9.3 compatibility (lowercase `c` language name)
+
+## Version 1.1.5 ##
+
+2009-08-20:
+
+- Use `RSTRING_PTR()` and `RSTRING_LEN()` instead of `RSTRING()->ptr` and
+  `RSTRING()->len` for ruby1.9 compatibility (and `#define` them, if not
+  existent)
+
+2009-10-02:
+
+- Patches for compatibility with Microsoft Visual Studio
+
+2009-10-08:
+
+- Fixes to make utf8proc usable in C++ programs
+
+2009-10-16:
+
+## Version 1.1.4 ##
+
+2009-06-14:
+
+- replaced C++ style comments for compatibility reasons
+- added typecasts to suppress compiler warnings
+- removed redundant source files for ruby-gemfile generation
+
+2009-08-19:
+
+- Changed copyright notice for Public Software Group e. V.
+- Minor changes in the `README` file
+
+## Version 1.1.3 ##
+
+2008-10-04:
+
+- Added a function `utf8proc_version` returning a string containing the version
+  number of the library.
+- Included a target `libutf8proc.dylib` for MacOSX.
+
+2009-05-01:
+- PostgreSQL 8.3 compatibility (use of `SET_VARSIZE` macro)
+
+## Version 1.1.2 ##
+
+2007-07-25:
+
+- Fixed a serious bug in the data file generator, which caused characters
+  being treated incorrectly, when stripping default ignorable characters or
+  calculating grapheme cluster boundaries.
+
+## Version 1.1.1 ##
+
+2007-06-25:
+
+- Added a new PostgreSQL function `unistrip`, which behaves like `unifold`,
+  but also removes all character marks (e.g. accents).
+
+2007-07-22:
+
+- Changed license from BSD to MIT style.
+- Added a new function `utf8proc_codepoint_valid` to the C library.
+- Changed compiler flags in `Makefile` from `-g -O0` to `-O2`
+- The ruby script, which was used to build the `utf8proc_data.c` file, is now
+  included in the distribution.
+
+## Version 1.0.3 ##
+
+2007-03-16:
+
+- Fixed a bug in the ruby library, which caused an error, when splitting an
+  empty string at grapheme cluster boundaries (method `String#utf8chars`).
+
+## Version 1.0.2 ##
+
+2006-09-21:
+
+- included a check in `Integer#utf8`, which raises an exception, if the given
+  code-point is invalid because of being too high (this was missing yet)
+
+2006-12-26:
+
+- added support for PostgreSQL version 8.2
+
+## Version 1.0.1 ##
+
+2006-09-20:
+
+- included a gem file for the ruby version of the library
+
+Release of version 1.0.1
+
+## Version 1.0 ##
+
+2006-09-17:
+
+- added the `LUMP` option, which lumps certain characters together (see `lump.md`) (also used for the PostgreSQL `unifold` function)
+- added the `STRIPMARK` option, which strips marking characters (or marks of composed characters)
+- deprecated ruby method `String#char_ary` in favour of `String#utf8chars`
+
+## Version 0.3 ##
+
+2006-07-18:
+
+- changed normalization from NFC to NFKC for postgresql unifold function
+
+2006-08-04:
+
+- added support to mark the beginning of a grapheme cluster with 0xFF (option: `CHARBOUND`)
+- added the ruby method `String#chars`, which is returning an array of UTF-8 encoded grapheme clusters
+- added `NLF2LF` transformation in postgresql `unifold` function
+- added the `DECOMPOSE` option, if you neither use `COMPOSE` or `DECOMPOSE`, no normalization will be performed (different from previous versions)
+- using integer constants rather than C-strings for character properties
+- fixed (hopefully) a problem with the ruby library on Mac OS X, which occurred when compiler optimization was switched on
+
+## Version 0.2 ##
+
+2006-06-05:
+
+- changed behaviour of PostgreSQL function to return NULL in case of invalid input, rather than raising an exceptional condition
+- improved efficiency of PostgreSQL function (no transformation to C string is done)
+
+2006-06-20:
+
+- added -fpic compiler flag in Makefile
+- fixed bug in the C code for the ruby library (usage of non-existent function)
+
+## Version 0.1 ##
+
+2006-06-02: initial release of version 0.1
+
+[#6]: https://github.com/JuliaLang/utf8proc/issues/6
+[#13]: https://github.com/JuliaLang/utf8proc/issues/13
+[#17]: https://github.com/JuliaLang/utf8proc/issues/17
+[#20]: https://github.com/JuliaLang/utf8proc/issues/20
+[#22]: https://github.com/JuliaLang/utf8proc/issues/22
+[#24]: https://github.com/JuliaLang/utf8proc/issues/24
+[#27]: https://github.com/JuliaLang/utf8proc/issues/27
+[#28]: https://github.com/JuliaLang/utf8proc/issues/28
+[#29]: https://github.com/JuliaLang/utf8proc/issues/29
+[#32]: https://github.com/JuliaLang/utf8proc/issues/32
+[#35]: https://github.com/JuliaLang/utf8proc/issues/35
+[#40]: https://github.com/JuliaLang/utf8proc/issues/40
+[#43]: https://github.com/JuliaLang/utf8proc/issues/43
+[#45]: https://github.com/JuliaLang/utf8proc/issues/45
+[#47]: https://github.com/JuliaLang/utf8proc/issues/47
+[#51]: https://github.com/JuliaLang/utf8proc/issues/51
+[#55]: https://github.com/JuliaLang/utf8proc/issues/55
+[#58]: https://github.com/JuliaLang/utf8proc/issues/58
+[#62]: https://github.com/JuliaLang/utf8proc/issues/62
+[#66]: https://github.com/JuliaLang/utf8proc/issues/66
+[#68]: https://github.com/JuliaLang/utf8proc/issues/68
+[#70]: https://github.com/JuliaLang/utf8proc/issues/70
+[#77]: https://github.com/JuliaLang/utf8proc/issues/77
+[#78]: https://github.com/JuliaLang/utf8proc/issues/78
+[#79]: https://github.com/JuliaLang/utf8proc/issues/79
+[#80]: https://github.com/JuliaLang/utf8proc/issues/80
+[#84]: https://github.com/JuliaLang/utf8proc/pull/84
+[#88]: https://github.com/JuliaLang/utf8proc/pull/88
+[#89]: https://github.com/JuliaLang/utf8proc/pull/89
+[#90]: https://github.com/JuliaLang/utf8proc/issues/90
--- a/lib/utf8proc-2a2f97e1/README.md
+++ b/lib/utf8proc-2a2f97e1/README.md
@ -0,0 +1,69 @@
+# utf8proc
+[![Travis CI Status](https://travis-ci.org/JuliaLang/utf8proc.png)](https://travis-ci.org/JuliaLang/utf8proc)
+[![AppVeyor Status](https://ci.appveyor.com/api/projects/status/aou20lfkyhj8xbwq/branch/master?svg=true)](https://ci.appveyor.com/project/tkelman/utf8proc/branch/master)
+
+
+[utf8proc](http://julialang.org/utf8proc/) is a small, clean C
+library that provides Unicode normalization, case-folding, and other
+operations for data in the [UTF-8
+encoding](http://en.wikipedia.org/wiki/UTF-8).  It was [initially
+developed](http://www.public-software-group.org/utf8proc) by Jan
+Behrens and the rest of the [Public Software
+Group](http://www.public-software-group.org/), who deserve *nearly all
+of the credit* for this package.  With the blessing of the Public
+Software Group, the [Julia developers](http://julialang.org/) have
+taken over development of utf8proc, since the original developers have
+moved to other projects.
+
+(utf8proc is used for basic Unicode
+support in the [Julia language](http://julialang.org/), and the Julia
+developers became involved because they wanted to add Unicode 7 support and other features.)
+
+(The original utf8proc package also includes Ruby and PostgreSQL plug-ins.
+We removed those from utf8proc in order to focus exclusively on the C
+library for the time being, but plan to add them back in or release them as separate packages.)
+
+The utf8proc package is licensed under the
+free/open-source [MIT "expat"
+license](http://opensource.org/licenses/MIT) (plus certain Unicode
+data governed by the similarly permissive [Unicode data
+license](http://www.unicode.org/copyright.html#Exhibit1)); please see
+the included `LICENSE.md` file for more detailed information.
+
+## Quick Start
+
+For compilation of the C library run `make`.
+
+## General Information
+
+The C library is found in this directory after successful compilation
+and is named `libutf8proc.a` (for the static library) and
+`libutf8proc.so` (for the dynamic library).
+
+The Unicode version supported is 9.0.0.
+
+For Unicode normalizations, the following options are used:
+
+* Normalization Form C:  `STABLE`, `COMPOSE`
+* Normalization Form D:  `STABLE`, `DECOMPOSE`
+* Normalization Form KC: `STABLE`, `COMPOSE`, `COMPAT`
+* Normalization Form KD: `STABLE`, `DECOMPOSE`, `COMPAT`
+
+## C Library
+
+The documentation for the C library is found in the `utf8proc.h` header file.
+`utf8proc_map` is function you will most likely be using for mapping UTF-8
+strings, unless you want to allocate memory yourself.
+
+## To Do
+
+See the Github [issues list](https://github.com/JuliaLang/utf8proc/issues).
+
+## Contact
+
+Bug reports, feature requests, and other queries can be filed at
+the [utf8proc issues page on Github](https://github.com/JuliaLang/utf8proc/issues).
+
+## See also
+
+An independent Lua translation of this library, [lua-mojibake](https://github.com/differentprogramming/lua-mojibake), is also available.
--- a/lib/utf8proc-2a2f97e1/appveyor.yml
+++ b/lib/utf8proc-2a2f97e1/appveyor.yml
@ -0,0 +1,42 @@
+branches:
+  only:
+    - master
+    - /release-.*/
+
+notifications:
+  - provider: Email
+    on_build_success: false
+    on_build_failure: false
+    on_build_status_changed: false
+
+build_script:
+  - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
+      https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
+      Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
+      throw "There are newer queued builds for this pull request, failing early." }
+  - mkdir msvc_static
+  - cd msvc_static
+  - cmake ..
+  - cmake --build .
+  - mkdir ..\msvc_shared
+  - cd ..\msvc_shared
+  - cmake .. -DBUILD_SHARED_LIBS=ON
+  - cmake --build .
+  - set PATH=C:\MinGW\bin;%PATH%
+  - C:\MinGW\msys\1.0\bin\sh --login -c "
+      cd /c/projects/utf8proc &&
+      mkdir mingw_static &&
+      cd mingw_static &&
+      cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -G'MSYS Makefiles' &&
+      make &&
+      mkdir ../mingw_shared &&
+      cd ../mingw_shared &&
+      cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DBUILD_SHARED_LIBS=ON -G'MSYS Makefiles' &&
+      make
+      "
+
+on_finish:
+  # Uncomment the following line for interactive debugging, which
+  # will print login data for a temporary remote session after the
+  # build. This requires an RDP version 6 client, e.g., FreeRDP.
+  #- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
--- a/lib/utf8proc-2a2f97e1/bench/Makefile
+++ b/lib/utf8proc-2a2f97e1/bench/Makefile
@ -0,0 +1,39 @@
+CURL=curl
+
+CC = cc
+CFLAGS = -O2 -std=c99 -pedantic -Wall
+
+all: bench
+
+LIBUTF8PROC = ../utf8proc.o
+
+bench: bench.o util.o $(LIBUTF8PROC)
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ bench.o util.o $(LIBUTF8PROC)
+
+DATAURL = https://raw.githubusercontent.com/duerst/eprun/master/benchmark
+DATAFILES = Deutsch_.txt Japanese_.txt Korean_.txt Vietnamese_.txt
+
+$(DATAFILES):
+	$(CURL) -O $(DATAURL)/$@
+
+bench.out: $(DATAFILES) bench
+	./bench -nfkc $(DATAFILES) > $@
+
+# you may need make CPPFLAGS=... LDFLAGS=... to help it find ICU
+icu: icu.o util.o
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ icu.o util.o -licuuc
+
+icu.out: $(DATAFILES) icu
+	./icu $(DATAFILES) > $@
+
+unistring: unistring.o util.o
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ unistring.o util.o -lunistring
+
+unistring.out: $(DATAFILES) unistring
+	./unistring $(DATAFILES) > $@
+
+.c.o:
+	$(CC) $(CPPFLAGS) -I.. $(CFLAGS) -c -o $@ $<
+
+clean:
+	rm -rf *.o *.txt bench *.out icu unistring
--- a/lib/utf8proc-2a2f97e1/bench/bench.c
+++ b/lib/utf8proc-2a2f97e1/bench/bench.c
@ -0,0 +1,56 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "utf8proc.h"
+#include "util.h"
+
+int main(int argc, char **argv)
+{
+	 int i, j;
+	 int options = 0;
+	 
+	 for (i = 1; i < argc; ++i) {
+		  if (!strcmp(argv[i], "-nfkc")) {
+			   options |= UTF8PROC_STABLE|UTF8PROC_COMPOSE|UTF8PROC_COMPAT;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfkd")) {
+			   options |= UTF8PROC_STABLE|UTF8PROC_DECOMPOSE|UTF8PROC_COMPAT;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfc")) {
+			   options |= UTF8PROC_STABLE|UTF8PROC_COMPOSE;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfd")) {
+			   options |= UTF8PROC_STABLE|UTF8PROC_DECOMPOSE;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-casefold")) {
+			   options |= UTF8PROC_CASEFOLD;
+			   continue;
+		  }
+		  if (argv[i][0] == '-') {
+			   fprintf(stderr, "unrecognized option: %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+
+		  size_t len;
+		  uint8_t *src = readfile(argv[i], &len);
+		  if (!src) {
+			   fprintf(stderr, "error reading %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+		  uint8_t *dest;
+		  mytime start = gettime();
+		  for (j = 0; j < 100; ++j) {
+			   utf8proc_map(src, len, &dest, options);
+			   free(dest);
+		  }
+		  printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
+		  free(src);
+	 }
+
+	 return EXIT_SUCCESS;
+}
--- a/lib/utf8proc-2a2f97e1/bench/icu.c
+++ b/lib/utf8proc-2a2f97e1/bench/icu.c
@ -0,0 +1,61 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+/* ICU4C */
+#include <unicode/utypes.h>
+#include <unicode/ustring.h>
+#include <unicode/ucnv.h>
+#include <unicode/unorm2.h>
+
+#include "util.h"
+
+int main(int argc, char **argv)
+{
+	 int i;
+
+	 UErrorCode err;
+	 UConverter *uc = ucnv_open("UTF8", &err);
+	 if (U_FAILURE(err)) return EXIT_FAILURE;
+
+	 const UNormalizer2 *NFKC = unorm2_getNFKCInstance(&err);
+	 if (U_FAILURE(err)) return EXIT_FAILURE;
+	 
+	 for (i = 1; i < argc; ++i) {
+		  if (argv[i][0] == '-') {
+			   fprintf(stderr, "unrecognized option: %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+
+		  size_t len;
+		  uint8_t *src = readfile(argv[i], &len);
+		  if (!src) {
+			   fprintf(stderr, "error reading %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+
+		  /* convert UTF8 data to ICU's UTF16 */
+		  UChar *usrc = (UChar*) malloc(2*len * sizeof(UChar));
+		  ucnv_toUChars(uc, usrc, 2*len, (char*) src, len, &err);
+		  if (U_FAILURE(err)) return EXIT_FAILURE;
+		  size_t ulen = u_strlen(usrc);
+
+		  /* ICU's insane normalization API requires you to
+			 know the size of the destination buffer in advance,
+			 or alternatively to repeatly try normalizing and
+			 double the buffer size until it succeeds.  Here, I just
+			 allocate a huge destination buffer to avoid the issue. */
+		  UChar *udest = (UChar*) malloc(10*ulen * sizeof(UChar));
+
+		  mytime start = gettime();
+		  for (int i = 0; i < 100; ++i) {
+			   unorm2_normalize(NFKC, usrc, ulen, udest, 10*ulen, &err);
+			   if (U_FAILURE(err)) return EXIT_FAILURE;
+		  }
+		  printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
+		  free(udest);
+		  free(usrc);
+		  free(src);
+	 }
+
+	 return EXIT_SUCCESS;
+}
--- a/lib/utf8proc-2a2f97e1/bench/unistring.c
+++ b/lib/utf8proc-2a2f97e1/bench/unistring.c
@ -0,0 +1,60 @@
+/* comparitive benchmark of GNU libunistring */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* libunistring */
+#include <unistr.h>
+#include <uninorm.h>
+
+#include "util.h"
+
+int main(int argc, char **argv)
+{
+	 int i;
+	 uninorm_t nf = UNINORM_NFKC;
+
+	 for (i = 1; i < argc; ++i) {
+		  if (!strcmp(argv[i], "-nfkc")) {
+			   nf = UNINORM_NFKC;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfkd")) {
+			   nf = UNINORM_NFKD;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfc")) {
+			   nf = UNINORM_NFC;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfd")) {
+			   nf = UNINORM_NFD;
+			   continue;
+		  }
+		  if (argv[i][0] == '-') {
+			   fprintf(stderr, "unrecognized option: %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+
+		  size_t len;
+		  uint8_t *src = readfile(argv[i], &len);
+		  if (!src) {
+			   fprintf(stderr, "error reading %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+
+		  size_t destlen;
+		  uint8_t *dest;
+		  mytime start = gettime();
+		  for (int i = 0; i < 100; ++i) {
+			   dest = u8_normalize(nf, src, len, NULL, &destlen);
+			   if (!dest) return EXIT_FAILURE;
+			   free(dest);
+		  }
+		  printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
+		  free(src);
+	 }
+
+	 return EXIT_SUCCESS;
+}
--- a/lib/utf8proc-2a2f97e1/bench/util.c
+++ b/lib/utf8proc-2a2f97e1/bench/util.c
@ -0,0 +1,39 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+
+#include "util.h"
+
+/* read file named FILENAME into an array of *len bytes,
+   returning NULL on error */
+uint8_t *readfile(const char *filename, size_t *len)
+{
+	 *len = 0;
+	 struct stat st;
+	 if (0 != stat(filename, &st)) return NULL;
+	 *len = st.st_size;
+	 FILE *f = fopen(filename, "r");
+	 if (!f) return NULL;
+	 uint8_t *s = (uint8_t *) malloc(sizeof(uint8_t) * *len);
+	 if (!s) return NULL;
+	 if (fread(s, 1, *len, f) != *len) {
+		  free(s);
+		  s = NULL;
+	 }
+	 fclose(f);
+	 return s;
+}
+
+mytime gettime(void) {
+	 mytime t;
+	 gettimeofday(&t, NULL);
+	 return t;
+}
+
+/* time difference in seconds */
+double elapsed(mytime t1, mytime t0)
+{
+     return (double)(t1.tv_sec - t0.tv_sec) +
+          (double)(t1.tv_usec - t0.tv_usec) * 1.0E-6;
+}
+
--- a/lib/utf8proc-2a2f97e1/bench/util.h
+++ b/lib/utf8proc-2a2f97e1/bench/util.h
@ -0,0 +1,22 @@
+#ifndef UTIL_H
+#define UTIL_H 1
+
+#include <inttypes.h>
+#include <sys/time.h>
+#include <time.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+uint8_t *readfile(const char *filename, size_t *len);
+
+typedef struct timeval mytime;
+mytime gettime(void);
+double elapsed(mytime t1, mytime t0);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* UTIL_H */
--- a/lib/utf8proc-2a2f97e1/data/Makefile
+++ b/lib/utf8proc-2a2f97e1/data/Makefile
@ -0,0 +1,66 @@
+# Unicode data generation rules.  Except for the test data files, most
+# users will not use these Makefile rules, which are primarily to re-generate
+# unicode_data.c when we get a new Unicode version or charwidth data; they
+# require ruby, fontforge, and julia to be installed.
+
+# programs
+CURL=curl
+RUBY=ruby
+PERL=perl
+MAKE=make
+JULIA=julia
+FONTFORGE=fontforge
+CURLFLAGS = --retry 5 --location
+
+.PHONY: clean
+
+.DELETE_ON_ERROR:
+
+utf8proc_data.c.new: data_generator.rb UnicodeData.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt CharWidths.txt
+	$(RUBY) data_generator.rb < UnicodeData.txt > $@
+
+# GNU Unifont version for font metric calculations:
+UNIFONT_VERSION=9.0.04
+
+unifont.ttf:
+	$(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont-$(UNIFONT_VERSION).ttf
+
+unifont_upper.ttf:
+	$(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont_upper-$(UNIFONT_VERSION).ttf
+
+%.sfd: %.ttf
+	$(FONTFORGE) -lang=ff -c "Open(\"$<\");Save(\"$@\");Quit(0);"
+
+CharWidths.txt: charwidths.jl unifont.sfd unifont_upper.sfd EastAsianWidth.txt
+	$(JULIA) charwidths.jl > $@
+
+# Unicode data version
+UNICODE_VERSION=9.0.0
+
+UnicodeData.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/UnicodeData.txt
+
+EastAsianWidth.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/EastAsianWidth.txt
+
+GraphemeBreakProperty.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/auxiliary/GraphemeBreakProperty.txt
+
+DerivedCoreProperties.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/DerivedCoreProperties.txt
+
+CompositionExclusions.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/CompositionExclusions.txt
+
+CaseFolding.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/CaseFolding.txt
+
+NormalizationTest.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/NormalizationTest.txt
+
+GraphemeBreakTest.txt:
+	$(CURL) $(CURLFLAGS) $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/auxiliary/GraphemeBreakTest.txt | $(PERL) -pe 's,÷,/,g;s,×,+,g' > $@
+
+clean:
+	rm -f UnicodeData.txt EastAsianWidth.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt NormalizationTest.txt GraphemeBreakTest.txt CharWidths.txt unifont*.ttf unifont*.sfd
+	rm -f utf8proc_data.c.new
--- a/lib/utf8proc-2a2f97e1/data/charwidths.jl
+++ b/lib/utf8proc-2a2f97e1/data/charwidths.jl
@ -0,0 +1,190 @@
+# Following work by @jiahao, we compute character widths using a combination of
+#   * advance widths from GNU Unifont (advance width 512 = 1 en)
+#   * UAX 11: East Asian Width
+#   * a few exceptions as needed
+# Adapted from http://nbviewer.ipython.org/gist/jiahao/07e8b08bf6d8671e9734
+#
+# Requires Julia (obviously) and FontForge.
+
+#############################################################################
+# Julia 0.3/0.4 compatibility (taken from Compat package)
+if VERSION < v"0.4.0-dev+1387"
+    typealias AbstractString String
+end
+if VERSION < v"0.4.0-dev+1419"
+    const UInt32 = Uint32
+end
+if VERSION < v"0.4.0-dev+3874"
+    Base.parse{T<:Integer}(::Type{T}, s::AbstractString) = parseint(T, s)
+end
+
+CharWidths = Dict{Int,Int}()
+
+#############################################################################
+# Use ../libutf8proc for category codes, rather than the one in Julia,
+# to minimize bootstrapping complexity when a new version of Unicode comes out.
+catcode(c) = ccall((:utf8proc_category,"../libutf8proc"), Cint, (Int32,), c)
+
+# use Base.UTF8proc module to get category codes constants, since
+# we won't change these in utf8proc.
+import Base.UTF8proc
+
+#############################################################################
+# Use a default width of 1 for all character categories that are
+# letter/symbol/number-like.  This can be overriden by Unifont or UAX 11
+# below, but provides a useful nonzero fallback for new codepoints when
+# a new Unicode version has been released but Unifont hasn't been updated yet.
+
+zerowidth = Set{Int}() # categories that may contain zero-width chars
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CN)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_MN)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_MC)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ME)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_SK)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZS)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZL)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZP)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CC)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CF)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CS)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CO)
+for c in 0x0000:0x110000
+    if catcode(c) ∉ zerowidth
+        CharWidths[c] = 1
+    end
+end
+
+#############################################################################
+# Widths from GNU Unifont
+
+#Read sfdfile for character widths
+function parsesfd(filename::AbstractString, CharWidths::Dict{Int,Int}=Dict{Int,Int}())
+    state=:seekchar
+    lineno = 0
+    codepoint = width = nothing
+    for line in readlines(open(filename))
+        lineno += 1
+        if state==:seekchar         #StartChar: nonmarkingreturn
+            if contains(line, "StartChar: ")
+                codepoint = nothing
+                width = nothing
+                state = :readdata
+            end
+        elseif state==:readdata #Encoding: 65538 -1 2, Width: 1024
+            contains(line, "Encoding:") && (codepoint = parse(Int, split(line)[3]))
+            contains(line, "Width:") && (width = parse(Int, split(line)[2]))
+            if codepoint!=nothing && width!=nothing && codepoint >= 0
+                w=div(width, 512) # 512 units to the en
+                if w > 0
+                    # only add nonzero widths, since (1) the default is zero
+                    # and (2) this circumvents some apparent bugs in Unifont
+                    # (https://savannah.gnu.org/bugs/index.php?45395)
+                    CharWidths[codepoint] = w
+                end
+                state = :seekchar
+            end
+        end
+    end
+    CharWidths
+end
+CharWidths=parsesfd("unifont.sfd", CharWidths)
+CharWidths=parsesfd("unifont_upper.sfd", CharWidths)
+
+#############################################################################
+# Widths from UAX #11: East Asian Width
+#   .. these take precedence over the Unifont width for all codepoints
+#      listed explicitly as wide/full/narrow/half-width
+
+for line in readlines(open("EastAsianWidth.txt"))
+    #Strip comments
+    line[1] == '#' && continue
+    precomment = split(line, '#')[1]
+    #Parse code point range and width code
+    tokens = split(precomment, ';')
+    length(tokens) >= 2 || continue
+    charrange = tokens[1]
+    width = strip(tokens[2])
+    #Parse code point range into Julia UnitRange
+    rangetokens = split(charrange, "..")
+    charstart = parse(UInt32, "0x"*rangetokens[1])
+    charend = parse(UInt32, "0x"*rangetokens[length(rangetokens)>1 ? 2 : 1])
+
+    #Assign widths
+    for c in charstart:charend
+        if width=="W" || width=="F" # wide or full
+            CharWidths[c]=2
+        elseif width=="Na"|| width=="H" # narrow or half
+            CharWidths[c]=1
+        end
+    end
+end
+
+#############################################################################
+# A few exceptions to the above cases, found by manual comparison
+# to other wcwidth functions and similar checks.
+
+for c in keys(CharWidths)
+    cat = catcode(c)
+
+    # make sure format control character (category Cf) have width 0,
+    # except for the Arabic characters 0x06xx (see unicode std 6.2, sec. 8.2)
+    if cat==UTF8proc.UTF8PROC_CATEGORY_CF && c ∉ [0x0601,0x0602,0x0603,0x06dd]
+        CharWidths[c]=0
+    end
+
+    # Unifont has nonzero width for a number of non-spacing combining
+    # characters, e.g. (in 7.0.06): f84,17b4,17b5,180b,180d,2d7f, and
+    # the variation selectors
+    if cat==UTF8proc.UTF8PROC_CATEGORY_MN
+        CharWidths[c]=0
+    end
+
+    # We also assign width of zero to unassigned and private-use
+    # codepoints (Unifont includes ConScript Unicode Registry PUA fonts,
+    # but since these are nonstandard it seems questionable to recognize them).
+    if cat==UTF8proc.UTF8PROC_CATEGORY_CO || cat==UTF8proc.UTF8PROC_CATEGORY_CN
+        CharWidths[c]=0
+    end
+
+    # for some reason, Unifont has width-2 glyphs for ASCII control chars
+    if cat==UTF8proc.UTF8PROC_CATEGORY_CC
+        CharWidths[c]=0
+    end
+end
+
+#By definition, should have zero width (on the same line)
+#0x002028 ' ' category: Zl name: LINE SEPARATOR/
+#0x002029 ' ' category: Zp name: PARAGRAPH SEPARATOR/
+CharWidths[0x2028]=0
+CharWidths[0x2029]=0
+
+#By definition, should be narrow = width of 1 en space
+#0x00202f ' ' category: Zs name: NARROW NO-BREAK SPACE/
+CharWidths[0x202f]=1
+
+#By definition, should be wide = width of 1 em space
+#0x002001 ' ' category: Zs name: EM QUAD/
+#0x002003 ' ' category: Zs name: EM SPACE/
+CharWidths[0x2001]=2
+CharWidths[0x2003]=2
+
+#############################################################################
+# Output (to a file or pipe) for processing by data_generator.rb
+# ... don't bother to output zero widths since that will be the default.
+
+firstc = 0x000000
+lastv = 0
+uhex(c) = uppercase(hex(c,4))
+for c in 0x0000:0x110000
+    v = get(CharWidths, c, 0)
+    if v != lastv || c == 0x110000
+        v < 4 || error("invalid charwidth $v for $c")
+        if firstc+1 < c
+            println(uhex(firstc), "..", uhex(c-1), "; ", lastv)
+        else
+            println(uhex(firstc), "; ", lastv)
+        end
+        firstc = c
+        lastv = v
+    end
+end
--- a/lib/utf8proc-2a2f97e1/data/data_generator.rb
+++ b/lib/utf8proc-2a2f97e1/data/data_generator.rb
@ -0,0 +1,411 @@
+#!/usr/bin/env ruby
+
+#  This file was used to generate the 'unicode_data.c' file by parsing the
+#  Unicode data file 'UnicodeData.txt' of the Unicode Character Database.
+#  It is included for informational purposes only and not intended for
+#  production use.
+
+
+#  Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a
+#  copy of this software and associated documentation files (the "Software"),
+#  to deal in the Software without restriction, including without limitation
+#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#  and/or sell copies of the Software, and to permit persons to whom the
+#  Software is furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#  DEALINGS IN THE SOFTWARE.
+
+
+#  This file contains derived data from a modified version of the
+#  Unicode data files. The following license applies to that data:
+#
+#  COPYRIGHT AND PERMISSION NOTICE
+#
+#  Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
+#  under the Terms of Use in http://www.unicode.org/copyright.html.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a
+#  copy of the Unicode data files and any associated documentation (the "Data
+#  Files") or Unicode software and any associated documentation (the
+#  "Software") to deal in the Data Files or Software without restriction,
+#  including without limitation the rights to use, copy, modify, merge,
+#  publish, distribute, and/or sell copies of the Data Files or Software, and
+#  to permit persons to whom the Data Files or Software are furnished to do
+#  so, provided that (a) the above copyright notice(s) and this permission
+#  notice appear with all copies of the Data Files or Software, (b) both the
+#  above copyright notice(s) and this permission notice appear in associated
+#  documentation, and (c) there is clear notice in each modified Data File or
+#  in the Software as well as in the documentation associated with the Data
+#  File(s) or Software that the data or software has been modified.
+#
+#  THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+#  KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+#  THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
+#  INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR
+#  CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
+#  USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+#  TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+#  PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+#
+#  Except as contained in this notice, the name of a copyright holder shall
+#  not be used in advertising or otherwise to promote the sale, use or other
+#  dealings in these Data Files or Software without prior written
+#  authorization of the copyright holder.
+
+
+$ignorable_list = File.read("DerivedCoreProperties.txt")[/# Derived Property: Default_Ignorable_Code_Point.*?# Total code points:/m]
+$ignorable = []
+$ignorable_list.each_line do |entry|
+  if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)/
+    $1.hex.upto($2.hex) { |e2| $ignorable << e2 }
+  elsif entry =~ /^[0-9A-F]+/
+    $ignorable << $&.hex
+  end
+end
+
+$grapheme_boundclass_list = File.read("GraphemeBreakProperty.txt")
+$grapheme_boundclass = Hash.new("UTF8PROC_BOUNDCLASS_OTHER")
+$grapheme_boundclass_list.each_line do |entry|
+  if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*([A-Za-z_]+)/
+    $1.hex.upto($2.hex) { |e2| $grapheme_boundclass[e2] = "UTF8PROC_BOUNDCLASS_" + $3.upcase }
+  elsif entry =~ /^([0-9A-F]+)\s*;\s*([A-Za-z_]+)/
+    $grapheme_boundclass[$1.hex] = "UTF8PROC_BOUNDCLASS_" + $2.upcase
+  end
+end
+
+$charwidth_list = File.read("CharWidths.txt")
+$charwidth = Hash.new(0)
+$charwidth_list.each_line do |entry|
+  if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*([0-9]+)/
+    $1.hex.upto($2.hex) { |e2| $charwidth[e2] = $3.to_i }
+  elsif entry =~ /^([0-9A-F]+)\s*;\s*([0-9]+)/
+    $charwidth[$1.hex] = $2.to_i
+  end
+end
+
+$exclusions = File.read("CompositionExclusions.txt")[/# \(1\) Script Specifics.*?# Total code points:/m]
+$exclusions = $exclusions.chomp.split("\n").collect { |e| e.hex }
+
+$excl_version = File.read("CompositionExclusions.txt")[/# \(2\) Post Composition Version precomposed characters.*?# Total code points:/m]
+$excl_version = $excl_version.chomp.split("\n").collect { |e| e.hex }
+
+$case_folding_string = File.open("CaseFolding.txt", :encoding => 'utf-8').read
+$case_folding = {}
+$case_folding_string.chomp.split("\n").each do |line|
+  next unless line =~ /([0-9A-F]+); [CFS]; ([0-9A-F ]+);/i
+  $case_folding[$1.hex] = $2.split(" ").collect { |e| e.hex }
+end
+
+$int_array = []
+$int_array_indicies = {}
+
+def str2c(string, prefix)
+  return "0" if string.nil?
+  return "UTF8PROC_#{prefix}_#{string.upcase}"
+end
+def pushary(array)
+  idx = $int_array_indicies[array]
+  unless idx
+    $int_array_indicies[array] = $int_array.length
+    idx = $int_array.length
+    array.each { |entry| $int_array << entry }
+  end
+  return idx
+end
+def cpary2utf16encoded(array)
+  return array.flat_map { |cp|
+      if (cp <= 0xFFFF)
+        raise "utf-16 code: #{cp}" if cp & 0b1111100000000000 == 0b1101100000000000
+        cp
+      else
+        temp = cp - 0x10000
+        [(temp >> 10) | 0b1101100000000000, (temp & 0b0000001111111111) | 0b1101110000000000]
+      end
+    }
+end
+def cpary2c(array)
+  return "UINT16_MAX" if array.nil? || array.length == 0
+  lencode = array.length - 1 #no sequence has len 0, so we encode len 1 as 0, len 2 as 1, ... 
+  array = cpary2utf16encoded(array)
+  if lencode >= 7 #we have only 3 bits for the length (which is already cutting it close. might need to change it to 2 bits in future Unicode versions)
+    array = [lencode] + array 
+    lencode = 7
+  end  
+  idx = pushary(array) 
+  raise "Array index out of bound" if idx > 0x1FFF
+  return "#{idx | (lencode << 13)}"
+end
+def singlecpmap(cp)
+  return "UINT16_MAX" if cp == nil
+  idx = pushary(cpary2utf16encoded([cp]))
+  raise "Array index out of bound" if idx > 0xFFFF
+  return "#{idx}"
+end
+
+class UnicodeChar
+  attr_accessor :code, :name, :category, :combining_class, :bidi_class,
+                :decomp_type, :decomp_mapping,
+                :bidi_mirrored,
+                :uppercase_mapping, :lowercase_mapping, :titlecase_mapping,
+                #caches:
+                :c_entry_index, :c_decomp_mapping, :c_case_folding
+  def initialize(line)
+    raise "Could not parse input." unless line =~ /^
+      ([0-9A-F]+);        # code
+      ([^;]+);            # name
+      ([A-Z]+);           # general category
+      ([0-9]+);           # canonical combining class
+      ([A-Z]+);           # bidi class
+      (<([A-Z]*)>)?       # decomposition type
+      ((\ ?[0-9A-F]+)*);  # decompomposition mapping
+      ([0-9]*);           # decimal digit
+      ([0-9]*);           # digit
+      ([^;]*);            # numeric
+      ([YN]*);            # bidi mirrored
+      ([^;]*);            # unicode 1.0 name
+      ([^;]*);            # iso comment
+      ([0-9A-F]*);        # simple uppercase mapping
+      ([0-9A-F]*);        # simple lowercase mapping
+      ([0-9A-F]*)$/ix     # simple titlecase mapping
+    @code              = $1.hex
+    @name              = $2
+    @category          = $3
+    @combining_class   = Integer($4)
+    @bidi_class        = $5
+    @decomp_type       = $7
+    @decomp_mapping    = ($8=='') ? nil :
+                         $8.split.collect { |element| element.hex }
+    @bidi_mirrored     = ($13=='Y') ? true : false
+    @uppercase_mapping = ($16=='') ? nil : $16.hex
+    @lowercase_mapping = ($17=='') ? nil : $17.hex
+    @titlecase_mapping = ($18=='') ? nil : $18.hex
+  end
+  def case_folding
+    $case_folding[code]
+  end
+  def c_entry(comb_indicies)
+    "  " <<
+    "{#{str2c category, 'CATEGORY'}, #{combining_class}, " <<
+    "#{str2c bidi_class, 'BIDI_CLASS'}, " <<
+    "#{str2c decomp_type, 'DECOMP_TYPE'}, " <<
+    "#{c_decomp_mapping}, " <<
+    "#{c_case_folding}, " <<
+    "#{singlecpmap uppercase_mapping }, " <<
+    "#{singlecpmap lowercase_mapping }, " <<
+    "#{singlecpmap titlecase_mapping }, " <<
+    "#{comb_indicies[code] ? comb_indicies[code]: 'UINT16_MAX'}, " <<
+    "#{bidi_mirrored}, " <<
+    "#{$exclusions.include?(code) or $excl_version.include?(code)}, " <<
+    "#{$ignorable.include?(code)}, " <<
+    "#{%W[Zl Zp Cc Cf].include?(category) and not [0x200C, 0x200D].include?(category)}, " <<
+    "#{$charwidth[code]}, 0, " <<
+    "#{$grapheme_boundclass[code]}},\n"
+  end
+end
+
+chars = []
+char_hash = {}
+
+while gets
+  if $_ =~ /^([0-9A-F]+);<[^;>,]+, First>;/i
+    first = $1.hex
+    gets
+    char = UnicodeChar.new($_)
+    raise "No last character of sequence found." unless
+      $_ =~ /^([0-9A-F]+);<([^;>,]+), Last>;/i
+    last = $1.hex
+    name = "<#{$2}>"
+    for i in first..last
+      char_clone = char.clone
+      char_clone.code = i
+      char_clone.name = name
+      char_hash[char_clone.code] = char_clone
+      chars << char_clone
+    end
+  else
+    char = UnicodeChar.new($_)
+    char_hash[char.code] = char
+    chars << char
+  end
+end
+
+comb1st_indicies = {}
+comb2nd_indicies = {}
+comb2nd_indicies_sorted_keys = []
+comb2nd_indicies_nonbasic = {}
+comb_array = []
+
+chars.each do |char|
+  if !char.nil? and char.decomp_type.nil? and char.decomp_mapping and
+      char.decomp_mapping.length == 2 and !char_hash[char.decomp_mapping[0]].nil? and
+      char_hash[char.decomp_mapping[0]].combining_class == 0 and
+      not $exclusions.include?(char.code)
+
+    dm0 = char.decomp_mapping[0]
+    dm1 = char.decomp_mapping[1]
+    unless comb1st_indicies[dm0]
+      comb1st_indicies[dm0] = comb1st_indicies.keys.length
+    end
+    unless comb2nd_indicies[dm1]
+      comb2nd_indicies_sorted_keys << dm1
+      comb2nd_indicies[dm1] = comb2nd_indicies.keys.length 
+    end
+    comb_array[comb1st_indicies[dm0]] ||= []
+    raise "Duplicate canonical mapping: #{char.code} #{dm0} #{dm1}" if comb_array[comb1st_indicies[dm0]][comb2nd_indicies[dm1]]
+    comb_array[comb1st_indicies[dm0]][comb2nd_indicies[dm1]] = char.code
+    
+    comb2nd_indicies_nonbasic[dm1] = true if char.code > 0xFFFF
+  end
+  char.c_decomp_mapping = cpary2c(char.decomp_mapping)
+  char.c_case_folding = cpary2c(char.case_folding)
+end 
+
+comb_indicies = {}
+cumoffset = 0
+comb1st_indicies_lastoffsets = []
+comb1st_indicies_firstoffsets = []
+comb1st_indicies.each do |dm0, index|
+  first = nil
+  last = nil
+  offset = 0
+  comb2nd_indicies_sorted_keys.each_with_index do |dm1, b|
+    if comb_array[index][b] 
+      first = offset unless first
+      last = offset
+      last += 1 if comb2nd_indicies_nonbasic[dm1]
+    end
+    offset += 1
+    offset += 1 if comb2nd_indicies_nonbasic[dm1]
+  end
+  comb1st_indicies_firstoffsets[index] = first
+  comb1st_indicies_lastoffsets[index] = last
+  raise "double index" if comb_indicies[dm0]
+  comb_indicies[dm0] = cumoffset
+  cumoffset += last - first + 1 + 2
+end
+
+offset = 0
+comb2nd_indicies_sorted_keys.each do |dm1|
+  raise "double index" if comb_indicies[dm1]
+  comb_indicies[dm1] = 0x8000 | (comb2nd_indicies[dm1] + offset)
+  raise "too large comb index" if  comb2nd_indicies[dm1] + offset > 0x4000
+  if comb2nd_indicies_nonbasic[dm1]
+    comb_indicies[dm1] = comb_indicies[dm1] | 0x4000
+    offset += 1
+  end
+end
+
+properties_indicies = {}
+properties = []
+chars.each do |char|
+  c_entry = char.c_entry(comb_indicies)
+  char.c_entry_index = properties_indicies[c_entry]
+  unless char.c_entry_index
+    properties_indicies[c_entry] = properties.length
+    char.c_entry_index = properties.length
+    properties << c_entry
+  end
+end
+
+stage1 = []
+stage2 = []
+for code in 0...0x110000
+  next unless code % 0x100 == 0
+  stage2_entry = []
+  for code2 in code...(code+0x100)
+    if char_hash[code2]
+      stage2_entry << (char_hash[code2].c_entry_index + 1)
+    else
+      stage2_entry << 0
+    end
+  end
+  old_index = stage2.index(stage2_entry)
+  if old_index
+    stage1 << (old_index * 0x100)
+  else
+    stage1 << (stage2.length * 0x100)
+    stage2 << stage2_entry
+  end
+end
+
+$stdout << "static const utf8proc_uint16_t utf8proc_sequences[] = {\n  "
+i = 0
+$int_array.each do |entry|
+  i += 1
+  if i == 8
+    i = 0
+    $stdout << "\n  "
+  end
+  $stdout << entry << ", "
+end
+$stdout << "};\n\n"
+
+$stdout << "static const utf8proc_uint16_t utf8proc_stage1table[] = {\n  "
+i = 0
+stage1.each do |entry|
+  i += 1
+  if i == 8
+    i = 0
+    $stdout << "\n  "
+  end
+  $stdout << entry << ", "
+end
+$stdout << "};\n\n"
+
+$stdout << "static const utf8proc_uint16_t utf8proc_stage2table[] = {\n  "
+i = 0
+stage2.flatten.each do |entry|
+  i += 1
+  if i == 8
+    i = 0
+    $stdout << "\n  "
+  end
+  $stdout << entry << ", "
+end
+$stdout << "};\n\n"
+
+$stdout << "static const utf8proc_property_t utf8proc_properties[] = {\n"
+$stdout << "  {0, 0, 0, 0, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,  false,false,false,false, 0, 0, UTF8PROC_BOUNDCLASS_OTHER},\n"
+properties.each { |line|
+  $stdout << line
+}
+$stdout << "};\n\n"
+
+
+
+$stdout << "static const utf8proc_uint16_t utf8proc_combinations[] = {\n  "
+i = 0
+comb1st_indicies.keys.each_index do |a|
+  offset = 0
+  $stdout << comb1st_indicies_firstoffsets[a] << ", " << comb1st_indicies_lastoffsets[a] << ", "
+  comb2nd_indicies_sorted_keys.each_with_index do |dm1, b|
+    break if offset > comb1st_indicies_lastoffsets[a] 
+    if offset >= comb1st_indicies_firstoffsets[a]
+      i += 1
+      if i == 8
+        i = 0
+        $stdout << "\n  "
+      end
+      v = comb_array[a][b] ? comb_array[a][b] : 0
+      $stdout << (( v & 0xFFFF0000 ) >> 16) << ", " if comb2nd_indicies_nonbasic[dm1]
+      $stdout << (v & 0xFFFF) << ", "
+    end
+    offset += 1
+    offset += 1 if comb2nd_indicies_nonbasic[dm1]    
+  end
+  $stdout  << "\n"
+end
+$stdout << "};\n\n"
+
--- a/lib/utf8proc-2a2f97e1/lump.md
+++ b/lib/utf8proc-2a2f97e1/lump.md
@ -0,0 +1,27 @@
+```
+U+0020      <-- all space characters (general category Zs)
+U+0027  '   <-- left/right single quotation mark U+2018..2019,
+                modifier letter apostrophe U+02BC,
+                modifier letter vertical line U+02C8
+U+002D  -   <-- all dash characters (general category Pd),
+                minus U+2212
+U+002F  /   <-- fraction slash U+2044,
+                division slash U+2215
+U+003A  :   <-- ratio U+2236
+U+003C  <   <-- single left-pointing angle quotation mark U+2039,
+                left-pointing angle bracket U+2329,
+                left angle bracket U+3008
+U+003E  >   <-- single right-pointing angle quotation mark U+203A,
+                right-pointing angle bracket U+232A,
+                right angle bracket U+3009
+U+005C  \   <-- set minus U+2216
+U+005E  ^   <-- modifier letter up arrowhead U+02C4,
+                modifier letter circumflex accent U+02C6,
+                caret U+2038,
+                up arrowhead U+2303
+U+005F  _   <-- all connector characters (general category Pc),
+                modifier letter low macron U+02CD
+U+0060  `   <-- modifier letter grave accent U+02CB
+U+007C  |   <-- divides U+2223
+U+007E  ~   <-- tilde operator U+223C
+```
--- a/lib/utf8proc-2a2f97e1/test/case.c
+++ b/lib/utf8proc-2a2f97e1/test/case.c
@ -0,0 +1,50 @@
+#include "tests.h"
+#include <wctype.h>
+
+int main(int argc, char **argv)
+{
+     int error = 0, better = 0;
+     utf8proc_int32_t c;
+
+     (void) argc; /* unused */
+     (void) argv; /* unused */
+
+     /* some simple sanity tests of the character widths */
+     for (c = 0; c <= 0x110000; ++c) {
+          utf8proc_int32_t l = utf8proc_tolower(c);
+          utf8proc_int32_t u = utf8proc_toupper(c);
+
+          check(l == c || utf8proc_codepoint_valid(l), "invalid tolower");
+          check(u == c || utf8proc_codepoint_valid(u), "invalid toupper");
+
+          if (sizeof(wint_t) > 2 || c < (1<<16)) {
+               wint_t l0 = towlower(c), u0 = towupper(c);
+               
+               /* OS unicode tables may be out of date.  But if they
+                  do have a lower/uppercase mapping, hopefully it
+                  is correct? */
+               if (l0 != c && l0 != l) {
+                    fprintf(stderr, "MISMATCH %x != towlower(%x) == %x\n",
+                            l, c, l0);
+                    ++error;
+               }
+               else if (l0 != l) { /* often true for out-of-date OS unicode */
+                    ++better;
+                    /* printf("%x != towlower(%x) == %x\n", l, c, l0); */
+               }
+               if (u0 != c && u0 != u) {
+                    fprintf(stderr, "MISMATCH %x != towupper(%x) == %x\n",
+                            u, c, u0);
+                    ++error;
+               }
+               else if (u0 != u) { /* often true for out-of-date OS unicode */
+                    ++better;
+                    /* printf("%x != towupper(%x) == %x\n", u, c, u0); */
+               }
+          }
+     }
+     check(!error, "utf8proc case conversion FAILED %d tests.", error);
+     printf("More up-to-date than OS unicode tables for %d tests.\n", better);
+     printf("utf8proc case conversion tests SUCCEEDED.\n");
+     return 0;
+}
--- a/lib/utf8proc-2a2f97e1/test/charwidth.c
+++ b/lib/utf8proc-2a2f97e1/test/charwidth.c
@ -0,0 +1,71 @@
+#include "tests.h"
+#include <ctype.h>
+#include <wchar.h>
+
+static int my_isprint(int c) {
+     int cat = utf8proc_get_property(c)->category;
+     return (UTF8PROC_CATEGORY_LU <= cat && cat <= UTF8PROC_CATEGORY_ZS) ||
+          (c == 0x0601 || c == 0x0602 || c == 0x0603 || c == 0x06dd);
+}
+
+int main(int argc, char **argv)
+{
+     int c, error = 0, updates = 0;
+
+     (void) argc; /* unused */
+     (void) argv; /* unused */
+
+     /* some simple sanity tests of the character widths */
+     for (c = 0; c <= 0x110000; ++c) {
+          int cat = utf8proc_get_property(c)->category;
+          int w = utf8proc_charwidth(c);
+          if ((cat == UTF8PROC_CATEGORY_MN || cat == UTF8PROC_CATEGORY_ME) &&
+              w > 0) {
+               fprintf(stderr, "nonzero width %d for combining char %x\n", w, c);
+               error = 1;
+          }
+          if (w == 0 &&
+			  ((cat >= UTF8PROC_CATEGORY_LU && cat <= UTF8PROC_CATEGORY_LO) ||
+			   (cat >= UTF8PROC_CATEGORY_ND && cat <= UTF8PROC_CATEGORY_SC) ||
+			   (cat >= UTF8PROC_CATEGORY_SO && cat <= UTF8PROC_CATEGORY_ZS))) {
+               fprintf(stderr, "zero width for symbol-like char %x\n", c);
+               error = 1;
+          }
+          if (c <= 127 && ((!isprint(c) && w > 0) ||
+                           (isprint(c) && wcwidth(c) != w))) {
+               fprintf(stderr, "wcwidth %d mismatch %d for %s ASCII %x\n",
+                       wcwidth(c), w, 
+                       isprint(c) ? "printable" : "non-printable", c);
+               error = 1;
+          }
+          if (!my_isprint(c) && w > 0) {
+               fprintf(stderr, "non-printing %x had width %d\n", c, w);
+               error = 1;
+          }
+     }
+     check(!error, "utf8proc_charwidth FAILED tests.");
+
+     /* print some other information by compariing with system wcwidth */
+     printf("Mismatches with system wcwidth (not necessarily errors):\n");
+     for (c = 0; c <= 0x110000; ++c) {
+          int w = utf8proc_charwidth(c);
+          int wc = wcwidth(c);
+          if (sizeof(wchar_t) == 2 && c >= (1<<16)) continue;
+          /* lots of these errors for out-of-date system unicode tables */
+          if (wc == -1 && my_isprint(c) && w > 0) {
+			   updates += 1;
+#if 0
+               printf("  wcwidth(%x) = -1 for printable char\n", c);
+#endif
+		  }
+          if (wc == -1 && !my_isprint(c) && w > 0)
+               printf("  wcwidth(%x) = -1 for non-printable width-%d char\n", c, w);
+          if (wc >= 0 && wc != w)
+               printf("  wcwidth(%x) = %d != charwidth %d\n", c, wc, w);
+     }
+	 printf("   ... (positive widths for %d chars unknown to wcwidth) ...\n",
+			updates);
+     printf("Character-width tests SUCCEEDED.\n");
+
+     return 0;
+}
--- a/lib/utf8proc-2a2f97e1/test/custom.c
+++ b/lib/utf8proc-2a2f97e1/test/custom.c
@ -0,0 +1,27 @@
+#include "tests.h"
+
+static int thunk_test = 1;
+
+static utf8proc_int32_t custom(utf8proc_int32_t codepoint, void *thunk)
+{
+    check(((int *) thunk) == &thunk_test, "unexpected thunk passed");
+    if (codepoint == 'a')
+        return 'b';
+    if (codepoint == 'S')
+        return 0x00df; /* ß */
+    return codepoint;
+}
+
+int main(void)
+{
+    utf8proc_uint8_t input[] = {0x41,0x61,0x53,0x62,0xef,0xbd,0x81,0x00}; /* "AaSb\uff41" */
+    utf8proc_uint8_t correct[] = {0x61,0x62,0x73,0x73,0x62,0x61,0x00}; /* "abssba" */
+    utf8proc_uint8_t *output;
+    utf8proc_map_custom(input, 0, &output, UTF8PROC_CASEFOLD | UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_NULLTERM,
+                        custom, &thunk_test);
+    printf("mapped \"%s\" -> \"%s\"\n", (char*)input, (char*)output);
+    check(strlen((char*) output) == 6, "incorrect output length");
+    check(!memcmp(correct, output, 7), "incorrect output data");
+    free(output);
+    return 0;
+}
--- a/lib/utf8proc-2a2f97e1/test/graphemetest.c
+++ b/lib/utf8proc-2a2f97e1/test/graphemetest.c
@ -0,0 +1,74 @@
+#include "tests.h"
+
+int main(int argc, char **argv)
+{
+    char *buf = NULL;
+    size_t bufsize = 0;
+    FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
+    utf8proc_uint8_t src[1024];
+    int len;
+    
+    check(f != NULL, "error opening GraphemeBreakTest.txt");
+    while (getline(&buf, &bufsize, f) > 0) {
+        size_t bi = 0, si = 0;
+        lineno += 1;
+        
+        if (lineno % 100 == 0)
+            printf("checking line %zd...\n", lineno);
+        
+        if (buf[0] == '#') continue;
+        
+        while (buf[bi]) {
+            bi = skipspaces(buf, bi);
+            if (buf[bi] == '/') { /* grapheme break */
+                src[si++] = '/';
+                bi++;
+            }
+            else if (buf[bi] == '+') { /* no break */
+                bi++;
+            }
+            else if (buf[bi] == '#') { /* start of comments */
+                break;
+            }
+	    else { /* hex-encoded codepoint */
+                len = encode((char*) (src + si), buf + bi) - 1;
+                while (src[si]) ++si; /* advance to NUL termination */
+                bi += len;
+            }
+        }
+        if (si && src[si-1] == '/')
+            --si; /* no break after final grapheme */
+        src[si] = 0; /* NUL-terminate */
+        
+        if (si) {
+            utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */
+            size_t i = 0, j = 0;
+            utf8proc_ssize_t glen;
+            utf8proc_uint8_t *g; /* utf8proc_map grapheme results */
+            while (i < si) {
+                if (src[i] != '/')
+                    utf8[j++] = src[i++];
+                else
+                    i++;
+            }
+            glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND);
+            if (glen == UTF8PROC_ERROR_INVALIDUTF8) {
+                 /* the test file contains surrogate codepoints, which are only for UTF-16 */
+                 printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno);
+            }
+            else {
+                 check(glen >= 0, "utf8proc_map error = %s",
+                       utf8proc_errmsg(glen));
+                 for (i = 0; i <= glen; ++i)
+                      if (g[i] == 0xff)
+                          g[i] = '/'; /* easier-to-read output (/ is not in test strings) */
+                 check(!strcmp((char*)g, (char*)src),
+                       "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src);
+            }
+            free(g);
+        }
+    }
+    fclose(f);
+    printf("Passed tests after %zd lines!\n", lineno);
+    return 0;
+}
--- a/lib/utf8proc-2a2f97e1/test/iterate.c
+++ b/lib/utf8proc-2a2f97e1/test/iterate.c
@ -0,0 +1,166 @@
+#include "tests.h"
+#include <ctype.h>
+#include <wchar.h>
+
+static  int     tests;
+static  int     error;
+
+#define CHECKVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,len,__LINE__)
+#define CHECKINVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,UTF8PROC_ERROR_INVALIDUTF8,__LINE__)
+
+static void testbytes(unsigned char *buf, int len, utf8proc_ssize_t retval, int line)
+{
+    utf8proc_int32_t out[16];
+    utf8proc_ssize_t ret;
+
+    /* Make a copy to ensure that memory is left uninitialized after "len"
+     * bytes. This way, Valgrind can detect overreads.
+     */
+    unsigned char tmp[16];
+    memcpy(tmp, buf, len);
+
+    tests++;
+    if ((ret = utf8proc_iterate(tmp, len, out)) != retval) {
+        fprintf(stderr, "Failed (%d):", line);
+        for (int i = 0; i < len ; i++) {
+            fprintf(stderr, " 0x%02x", tmp[i]);
+        }
+        fprintf(stderr, " -> %zd\n", ret);
+        error++;
+    }
+}
+
+int main(int argc, char **argv)
+{
+    uint32_t byt;
+    unsigned char buf[16];
+
+    tests = error = 0;
+
+    // Check valid sequences that were considered valid erroneously before
+    buf[0] = 0xef;
+    buf[1] = 0xb7;
+    for (byt = 0x90; byt < 0xa0; byt++) {
+        CHECKVALID(2, byt, 3);
+    }
+    // Check 0xfffe and 0xffff
+    buf[1] = 0xbf;
+    CHECKVALID(2, 0xbe, 3);
+    CHECKVALID(2, 0xbf, 3);
+    // Check 0x??fffe & 0x??ffff
+    for (byt = 0x1fffe; byt < 0x110000; byt += 0x10000) {
+        buf[0] = 0xf0 | (byt >> 18);
+        buf[1] = 0x80 | ((byt >> 12) & 0x3f);
+        CHECKVALID(3, 0xbe, 4);
+        CHECKVALID(3, 0xbf, 4);
+    }
+    
+    // Continuation byte not after lead
+    for (byt = 0x80; byt < 0xc0; byt++) {
+        CHECKINVALID(0, byt, 1);
+    }
+
+    // Continuation byte not after lead
+    for (byt = 0x80; byt < 0xc0; byt++) {
+        CHECKINVALID(0, byt, 1);
+    }
+
+    // Test lead bytes
+    for (byt = 0xc0; byt <= 0xff; byt++) {
+	// Single lead byte at end of string
+        CHECKINVALID(0, byt, 1);
+        // Lead followed by non-continuation character < 0x80
+        CHECKINVALID(1, 65, 2);
+	// Lead followed by non-continuation character > 0xbf
+        CHECKINVALID(1, 0xc0, 2);
+    }
+
+    // Test overlong 2-byte
+    buf[0] = 0xc0;
+    for (byt = 0x81; byt <= 0xbf; byt++) {
+        CHECKINVALID(1, byt, 2);
+    }
+    buf[0] = 0xc1;
+    for (byt = 0x80; byt <= 0xbf; byt++) {
+        CHECKINVALID(1, byt, 2);
+    }
+
+    // Test overlong 3-byte
+    buf[0] = 0xe0;
+    buf[2] = 0x80;
+    for (byt = 0x80; byt <= 0x9f; byt++) {
+        CHECKINVALID(1, byt, 3);
+    }
+
+    // Test overlong 4-byte
+    buf[0] = 0xf0;
+    buf[2] = 0x80;
+    buf[3] = 0x80;
+    for (byt = 0x80; byt <= 0x8f; byt++) {
+        CHECKINVALID(1, byt, 4);
+    }
+
+    // Test 4-byte > 0x10ffff
+    buf[0] = 0xf4;
+    buf[2] = 0x80;
+    buf[3] = 0x80;
+    for (byt = 0x90; byt <= 0xbf; byt++) {
+        CHECKINVALID(1, byt, 4);
+    }
+    buf[1] = 0x80;
+    for (byt = 0xf5; byt <= 0xf7; byt++) {
+        CHECKINVALID(0, byt, 4);
+    }
+
+    // Test 5-byte
+    buf[4] = 0x80;
+    for (byt = 0xf8; byt <= 0xfb; byt++) {
+        CHECKINVALID(0, byt, 5);
+    }
+
+    // Test 6-byte
+    buf[5] = 0x80;
+    for (byt = 0xfc; byt <= 0xfd; byt++) {
+        CHECKINVALID(0, byt, 6);
+    }
+
+    // Test 7-byte
+    buf[6] = 0x80;
+    CHECKINVALID(0, 0xfe, 7);
+
+    // Three and above byte sequences
+    for (byt = 0xe0; byt < 0xf0; byt++) {
+        // Lead followed by only 1 continuation byte
+        CHECKINVALID(0, byt, 2);
+        // Lead ended by non-continuation character < 0x80
+        CHECKINVALID(2, 65, 3);
+        // Lead ended by non-continuation character > 0xbf
+        CHECKINVALID(2, 0xc0, 3);
+    }
+
+    // 3-byte encoded surrogate character(s)
+    buf[0] = 0xed; buf[2] = 0x80;
+    // Single surrogate
+    CHECKINVALID(1, 0xa0, 3);
+    // Trailing surrogate first
+    CHECKINVALID(1, 0xb0, 3);
+
+    // Four byte sequences
+    buf[1] = 0x80;
+    for (byt = 0xf0; byt < 0xf5; byt++) {
+        // Lead followed by only 1 continuation bytes
+        CHECKINVALID(0, byt, 2);
+        // Lead followed by only 2 continuation bytes
+        CHECKINVALID(0, byt, 3);
+        // Lead followed by non-continuation character < 0x80
+        CHECKINVALID(3, 65, 4);
+        // Lead followed by non-continuation character > 0xbf
+        CHECKINVALID(3, 0xc0, 4);
+
+    }
+
+     check(!error, "utf8proc_iterate FAILED %d tests out of %d", error, tests);
+     printf("utf8proc_iterate tests SUCCEEDED, (%d) tests passed.\n", tests);
+
+     return 0;
+}
--- a/lib/utf8proc-2a2f97e1/test/normtest.c
+++ b/lib/utf8proc-2a2f97e1/test/normtest.c
@ -0,0 +1,64 @@
+#include "tests.h"
+
+#define CHECK_NORM(NRM, norm, src) {                                 \
+    char *src_norm = (char*) utf8proc_ ## NRM((utf8proc_uint8_t*) src);      \
+    check(!strcmp(norm, src_norm),                                  \
+          "normalization failed for %s -> %s", src, norm);          \
+    free(src_norm);                                                 \
+}
+
+int main(int argc, char **argv)
+{
+     char *buf = NULL;
+     size_t bufsize = 0;
+     FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
+     char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024];
+
+     check(f != NULL, "error opening NormalizationTest.txt");
+     while (getline(&buf, &bufsize, f) > 0) {
+          size_t offset;
+          lineno += 1;
+
+          if (buf[0] == '@') {
+               printf("line %zd: %s", lineno, buf + 1);
+               continue;
+          }
+          else if (lineno % 1000 == 0)
+               printf("checking line %zd...\n", lineno);
+
+          if (buf[0] == '#') continue;
+
+          offset = encode(source, buf);
+          offset += encode(NFC, buf + offset);
+          offset += encode(NFD, buf + offset);
+          offset += encode(NFKC, buf + offset);
+          offset += encode(NFKD, buf + offset);
+
+          CHECK_NORM(NFC, NFC, source);
+          CHECK_NORM(NFC, NFC, NFC);
+          CHECK_NORM(NFC, NFC, NFD);
+          CHECK_NORM(NFC, NFKC, NFKC);
+          CHECK_NORM(NFC, NFKC, NFKD);
+
+          CHECK_NORM(NFD, NFD, source);
+          CHECK_NORM(NFD, NFD, NFC);
+          CHECK_NORM(NFD, NFD, NFD);
+          CHECK_NORM(NFD, NFKD, NFKC);
+          CHECK_NORM(NFD, NFKD, NFKD);
+
+          CHECK_NORM(NFKC, NFKC, source);
+          CHECK_NORM(NFKC, NFKC, NFC);
+          CHECK_NORM(NFKC, NFKC, NFD);
+          CHECK_NORM(NFKC, NFKC, NFKC);
+          CHECK_NORM(NFKC, NFKC, NFKD);
+
+          CHECK_NORM(NFKD, NFKD, source);
+          CHECK_NORM(NFKD, NFKD, NFC);
+          CHECK_NORM(NFKD, NFKD, NFD);
+          CHECK_NORM(NFKD, NFKD, NFKC);
+          CHECK_NORM(NFKD, NFKD, NFKD);
+     }
+     fclose(f);
+     printf("Passed tests after %zd lines!\n", lineno);
+     return 0;
+}
--- a/lib/utf8proc-2a2f97e1/test/printproperty.c
+++ b/lib/utf8proc-2a2f97e1/test/printproperty.c
@ -0,0 +1,49 @@
+/* simple test program to print out the utf8proc properties for a codepoint */
+
+#include "tests.h"
+
+int main(int argc, char **argv)
+{
+     int i;
+
+     for (i = 1; i < argc; ++i) {
+          unsigned int c;
+          if (!strcmp(argv[i], "-V")) {
+               printf("utf8proc version %s\n", utf8proc_version());
+               continue;
+          }
+          check(sscanf(argv[i],"%x",&c) == 1, "invalid hex input %s", argv[i]);
+          const utf8proc_property_t *p = utf8proc_get_property(c);
+          printf("U+%s:\n"
+                 "  category = %s\n"
+                 "  combining_class = %d\n"
+                 "  bidi_class = %d\n"
+                 "  decomp_type = %d\n"
+                 "  uppercase_mapping = %x\n"
+                 "  lowercase_mapping = %x\n"
+                 "  titlecase_mapping = %x\n"
+                 "  comb_index = %d\n"
+                 "  bidi_mirrored = %d\n"
+                 "  comp_exclusion = %d\n"
+                 "  ignorable = %d\n"
+                 "  control_boundary = %d\n"
+                 "  boundclass = %d\n"
+                 "  charwidth = %d\n",
+                 argv[i],
+                 utf8proc_category_string(c),
+                 p->combining_class,
+                 p->bidi_class,
+                 p->decomp_type,
+                 utf8proc_toupper(c),
+                 utf8proc_tolower(c),
+                 utf8proc_totitle(c),
+                 p->comb_index,
+                 p->bidi_mirrored,
+                 p->comp_exclusion,
+                 p->ignorable,
+                 p->control_boundary,
+                 p->boundclass,
+                 utf8proc_charwidth(c));
+     }
+     return 0;
+}
--- a/lib/utf8proc-2a2f97e1/test/tests.c
+++ b/lib/utf8proc-2a2f97e1/test/tests.c
@ -0,0 +1,46 @@
+/* Common functions for our test programs. */
+
+#include "tests.h"
+
+size_t lineno = 0;
+
+void check(int cond, const char *format, ...)
+{
+     if (!cond) {
+          va_list args;
+          fprintf(stderr, "line %zd: ", lineno);
+          va_start(args, format);
+          vfprintf(stderr, format, args);
+          va_end(args);
+          fprintf(stderr, "\n");
+          exit(1);
+     }
+}
+
+size_t skipspaces(const char *buf, size_t i)
+{
+    while (isspace(buf[i])) ++i;
+    return i;
+}
+
+/* if buf points to a sequence of codepoints encoded as hexadecimal strings,
+   separated by whitespace, and terminated by any character not in
+   [0-9a-fA-F] or whitespace, then stores the corresponding utf8 string
+   in dest, returning the number of bytes read from buf */
+size_t encode(char *dest, const char *buf)
+{
+     size_t i = 0, j, d = 0;
+     for (;;) {
+          int c;
+          i = skipspaces(buf, i);
+          for (j=i; buf[j] && strchr("0123456789abcdef", tolower(buf[j])); ++j)
+               ; /* find end of hex input */
+          if (j == i) { /* no codepoint found */
+               dest[d] = 0; /* NUL-terminate destination string */
+               return i + 1;
+          }
+          check(sscanf(buf + i, "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i);
+          i = j; /* skip to char after hex input */
+          d += utf8proc_encode_char(c, (utf8proc_uint8_t *) (dest + d));
+     }
+}
--- a/lib/utf8proc-2a2f97e1/test/tests.h
+++ b/lib/utf8proc-2a2f97e1/test/tests.h
@ -0,0 +1,23 @@
+/* Common functions and includes for our test programs. */
+
+/*
+ * Set feature macro to enable getline() and wcwidth().
+ *
+ * Please refer to section 2.2.1 of POSIX.1-2008:
+ * http://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_02_01_02
+ */
+#define _XOPEN_SOURCE 700
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+#include <stdarg.h>
+
+#include "../utf8proc.h"
+
+extern size_t lineno;
+
+void check(int cond, const char *format, ...);
+size_t skipspaces(const char *buf, size_t i);
+size_t encode(char *dest, const char *buf);
--- a/lib/utf8proc-2a2f97e1/test/valid.c
+++ b/lib/utf8proc-2a2f97e1/test/valid.c
@ -0,0 +1,41 @@
+#include "tests.h"
+#include <ctype.h>
+#include <wchar.h>
+
+int main(int argc, char **argv)
+{
+     int c, error = 0;
+
+     (void) argc; /* unused */
+     (void) argv; /* unused */
+
+     /* some simple sanity tests of  */
+     for (c = 0; c < 0xd800; c++) {
+        if (!utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%04x) -> false\n", c);
+           error++;
+        }
+     }
+     for (;c < 0xe000; c++) {
+        if (utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%04x) -> true\n", c);
+           error++;
+        }
+     }
+     for (;c < 0x110000; c++) {
+        if (!utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%06x) -> false\n", c);
+           error++;
+        }
+     }
+     for (;c < 0x110010; c++) {
+        if (utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%06x) -> true\n", c);
+           error++;
+        }
+     }
+     check(!error, "utf8proc_codepoint_valid FAILED %d tests.", error);
+     printf("Validity tests SUCCEEDED.\n");
+
+     return 0;
+}
--- a/lib/utf8proc-2a2f97e1/utf8proc.c
+++ b/lib/utf8proc-2a2f97e1/utf8proc.c
@ -0,0 +1,755 @@
+/* -*- mode: c; c-basic-offset: 2; tab-width: 2; indent-tabs-mode: nil -*- */
+/*
+ *  Copyright (c) 2015 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
+ *  Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a
+ *  copy of this software and associated documentation files (the "Software"),
+ *  to deal in the Software without restriction, including without limitation
+ *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ *  Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ *  DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ *  This library contains derived data from a modified version of the
+ *  Unicode data files.
+ *
+ *  The original data files are available at
+ *  http://www.unicode.org/Public/UNIDATA/
+ *
+ *  Please notice the copyright statement in the file "utf8proc_data.c".
+ */
+
+
+/*
+ *  File name:    utf8proc.c
+ *
+ *  Description:
+ *  Implementation of libutf8proc.
+ */
+
+
+#include "utf8proc.h"
+#include "utf8proc_data.c"
+
+
+UTF8PROC_DLLEXPORT const utf8proc_int8_t utf8proc_utf8class[256] = {
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+  3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+  4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0 };
+
+#define UTF8PROC_HANGUL_SBASE 0xAC00
+#define UTF8PROC_HANGUL_LBASE 0x1100
+#define UTF8PROC_HANGUL_VBASE 0x1161
+#define UTF8PROC_HANGUL_TBASE 0x11A7
+#define UTF8PROC_HANGUL_LCOUNT 19
+#define UTF8PROC_HANGUL_VCOUNT 21
+#define UTF8PROC_HANGUL_TCOUNT 28
+#define UTF8PROC_HANGUL_NCOUNT 588
+#define UTF8PROC_HANGUL_SCOUNT 11172
+/* END is exclusive */
+#define UTF8PROC_HANGUL_L_START  0x1100
+#define UTF8PROC_HANGUL_L_END    0x115A
+#define UTF8PROC_HANGUL_L_FILLER 0x115F
+#define UTF8PROC_HANGUL_V_START  0x1160
+#define UTF8PROC_HANGUL_V_END    0x11A3
+#define UTF8PROC_HANGUL_T_START  0x11A8
+#define UTF8PROC_HANGUL_T_END    0x11FA
+#define UTF8PROC_HANGUL_S_START  0xAC00
+#define UTF8PROC_HANGUL_S_END    0xD7A4
+
+/* Should follow semantic-versioning rules (semver.org) based on API
+   compatibility.  (Note that the shared-library version number will
+   be different, being based on ABI compatibility.): */
+#define STRINGIZEx(x) #x
+#define STRINGIZE(x) STRINGIZEx(x)
+UTF8PROC_DLLEXPORT const char *utf8proc_version(void) {
+  return STRINGIZE(UTF8PROC_VERSION_MAJOR) "." STRINGIZE(UTF8PROC_VERSION_MINOR) "." STRINGIZE(UTF8PROC_VERSION_PATCH) "";
+}
+
+UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode) {
+  switch (errcode) {
+    case UTF8PROC_ERROR_NOMEM:
+    return "Memory for processing UTF-8 data could not be allocated.";
+    case UTF8PROC_ERROR_OVERFLOW:
+    return "UTF-8 string is too long to be processed.";
+    case UTF8PROC_ERROR_INVALIDUTF8:
+    return "Invalid UTF-8 string";
+    case UTF8PROC_ERROR_NOTASSIGNED:
+    return "Unassigned Unicode code point found in UTF-8 string.";
+    case UTF8PROC_ERROR_INVALIDOPTS:
+    return "Invalid options for UTF-8 processing chosen.";
+    default:
+    return "An unknown error occurred while processing UTF-8 data.";
+  }
+}
+
+#define utf_cont(ch)  (((ch) & 0xc0) == 0x80)
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *dst
+) {
+  utf8proc_uint32_t uc;
+  const utf8proc_uint8_t *end;
+
+  *dst = -1;
+  if (!strlen) return 0;
+  end = str + ((strlen < 0) ? 4 : strlen);
+  uc = *str++;
+  if (uc < 0x80) {
+    *dst = uc;
+    return 1;
+  }
+  // Must be between 0xc2 and 0xf4 inclusive to be valid
+  if ((uc - 0xc2) > (0xf4-0xc2)) return UTF8PROC_ERROR_INVALIDUTF8;
+  if (uc < 0xe0) {         // 2-byte sequence
+     // Must have valid continuation character
+     if (str >= end || !utf_cont(*str)) return UTF8PROC_ERROR_INVALIDUTF8;
+     *dst = ((uc & 0x1f)<<6) | (*str & 0x3f);
+     return 2;
+  }
+  if (uc < 0xf0) {        // 3-byte sequence
+     if ((str + 1 >= end) || !utf_cont(*str) || !utf_cont(str[1]))
+        return UTF8PROC_ERROR_INVALIDUTF8;
+     // Check for surrogate chars
+     if (uc == 0xed && *str > 0x9f)
+         return UTF8PROC_ERROR_INVALIDUTF8;
+     uc = ((uc & 0xf)<<12) | ((*str & 0x3f)<<6) | (str[1] & 0x3f);
+     if (uc < 0x800)
+         return UTF8PROC_ERROR_INVALIDUTF8;
+     *dst = uc;
+     return 3;
+  }
+  // 4-byte sequence
+  // Must have 3 valid continuation characters
+  if ((str + 2 >= end) || !utf_cont(*str) || !utf_cont(str[1]) || !utf_cont(str[2]))
+     return UTF8PROC_ERROR_INVALIDUTF8;
+  // Make sure in correct range (0x10000 - 0x10ffff)
+  if (uc == 0xf0) {
+    if (*str < 0x90) return UTF8PROC_ERROR_INVALIDUTF8;
+  } else if (uc == 0xf4) {
+    if (*str > 0x8f) return UTF8PROC_ERROR_INVALIDUTF8;
+  }
+  *dst = ((uc & 7)<<18) | ((*str & 0x3f)<<12) | ((str[1] & 0x3f)<<6) | (str[2] & 0x3f);
+  return 4;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t uc) {
+    return (((utf8proc_uint32_t)uc)-0xd800 > 0x07ff) && ((utf8proc_uint32_t)uc < 0x110000);
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
+  if (uc < 0x00) {
+    return 0;
+  } else if (uc < 0x80) {
+    dst[0] = (utf8proc_uint8_t) uc;
+    return 1;
+  } else if (uc < 0x800) {
+    dst[0] = (utf8proc_uint8_t)(0xC0 + (uc >> 6));
+    dst[1] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+    return 2;
+  // Note: we allow encoding 0xd800-0xdfff here, so as not to change
+  // the API, however, these are actually invalid in UTF-8
+  } else if (uc < 0x10000) {
+    dst[0] = (utf8proc_uint8_t)(0xE0 + (uc >> 12));
+    dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
+    dst[2] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+    return 3;
+  } else if (uc < 0x110000) {
+    dst[0] = (utf8proc_uint8_t)(0xF0 + (uc >> 18));
+    dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 12) & 0x3F));
+    dst[2] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
+    dst[3] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+    return 4;
+  } else return 0;
+}
+
+/* internal "unsafe" version that does not check whether uc is in range */
+static utf8proc_ssize_t unsafe_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
+   if (uc < 0x00) {
+      return 0;
+   } else if (uc < 0x80) {
+      dst[0] = (utf8proc_uint8_t)uc;
+      return 1;
+   } else if (uc < 0x800) {
+      dst[0] = (utf8proc_uint8_t)(0xC0 + (uc >> 6));
+      dst[1] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+      return 2;
+   } else if (uc == 0xFFFF) {
+       dst[0] = (utf8proc_uint8_t)0xFF;
+       return 1;
+   } else if (uc == 0xFFFE) {
+       dst[0] = (utf8proc_uint8_t)0xFE;
+       return 1;
+   } else if (uc < 0x10000) {
+      dst[0] = (utf8proc_uint8_t)(0xE0 + (uc >> 12));
+      dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
+      dst[2] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+      return 3;
+   } else if (uc < 0x110000) {
+      dst[0] = (utf8proc_uint8_t)(0xF0 + (uc >> 18));
+      dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 12) & 0x3F));
+      dst[2] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
+      dst[3] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+      return 4;
+   } else return 0;
+}
+
+/* internal "unsafe" version that does not check whether uc is in range */
+static const utf8proc_property_t *unsafe_get_property(utf8proc_int32_t uc) {
+  /* ASSERT: uc >= 0 && uc < 0x110000 */
+  return utf8proc_properties + (
+    utf8proc_stage2table[
+      utf8proc_stage1table[uc >> 8] + (uc & 0xFF)
+    ]
+  );
+}
+
+UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t uc) {
+  return uc < 0 || uc >= 0x110000 ? utf8proc_properties : unsafe_get_property(uc);
+}
+
+/* return whether there is a grapheme break between boundclasses lbc and tbc
+   (according to the definition of extended grapheme clusters)
+
+  Rule numbering refers to TR29 Version 29 (Unicode 9.0.0):
+  http://www.unicode.org/reports/tr29/tr29-29.html
+
+  CAVEATS:
+   Please note that evaluation of GB10 (grapheme breaks between emoji zwj sequences)
+   and GB 12/13 (regional indicator code points) require knowledge of previous characters
+   and are thus not handled by this function. This may result in an incorrect break before
+   an E_Modifier class codepoint and an incorrectly missing break between two
+   REGIONAL_INDICATOR class code points if such support does not exist in the caller.
+
+   See the special support in grapheme_break_extended, for required bookkeeping by the caller.
+*/
+static utf8proc_bool grapheme_break_simple(int lbc, int tbc) {
+  return
+    (lbc == UTF8PROC_BOUNDCLASS_START) ? true :       // GB1
+    (lbc == UTF8PROC_BOUNDCLASS_CR &&                 // GB3
+     tbc == UTF8PROC_BOUNDCLASS_LF) ? false :         // ---
+    (lbc >= UTF8PROC_BOUNDCLASS_CR && lbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true :  // GB4
+    (tbc >= UTF8PROC_BOUNDCLASS_CR && tbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true :  // GB5
+    (lbc == UTF8PROC_BOUNDCLASS_L &&                  // GB6
+     (tbc == UTF8PROC_BOUNDCLASS_L ||                 // ---
+      tbc == UTF8PROC_BOUNDCLASS_V ||                 // ---
+      tbc == UTF8PROC_BOUNDCLASS_LV ||                // ---
+      tbc == UTF8PROC_BOUNDCLASS_LVT)) ? false :      // ---
+    ((lbc == UTF8PROC_BOUNDCLASS_LV ||                // GB7
+      lbc == UTF8PROC_BOUNDCLASS_V) &&                // ---
+     (tbc == UTF8PROC_BOUNDCLASS_V ||                 // ---
+      tbc == UTF8PROC_BOUNDCLASS_T)) ? false :        // ---
+    ((lbc == UTF8PROC_BOUNDCLASS_LVT ||               // GB8
+      lbc == UTF8PROC_BOUNDCLASS_T) &&                // ---
+     tbc == UTF8PROC_BOUNDCLASS_T) ? false :          // ---
+    (tbc == UTF8PROC_BOUNDCLASS_EXTEND ||             // GB9
+     tbc == UTF8PROC_BOUNDCLASS_ZWJ ||                // ---
+     tbc == UTF8PROC_BOUNDCLASS_SPACINGMARK ||        // GB9a
+     lbc == UTF8PROC_BOUNDCLASS_PREPEND) ? false :    // GB9b
+    ((lbc == UTF8PROC_BOUNDCLASS_E_BASE ||            // GB10 (requires additional handling below)
+      lbc == UTF8PROC_BOUNDCLASS_E_BASE_GAZ) &&       // ----
+     tbc == UTF8PROC_BOUNDCLASS_E_MODIFIER) ? false : // ----
+    (lbc == UTF8PROC_BOUNDCLASS_ZWJ &&                         // GB11
+     (tbc == UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ ||             // ----
+      tbc == UTF8PROC_BOUNDCLASS_E_BASE_GAZ)) ? false :        // ----
+    (lbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR &&          // GB12/13 (requires additional handling below)
+     tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR) ? false :  // ----
+    true; // GB999
+}
+
+static utf8proc_bool grapheme_break_extended(int lbc, int tbc, utf8proc_int32_t *state)
+{
+  int lbc_override = lbc;
+  if (state && *state != UTF8PROC_BOUNDCLASS_START)
+    lbc_override = *state;
+  utf8proc_bool break_permitted = grapheme_break_simple(lbc_override, tbc);
+  if (state) {
+    // Special support for GB 12/13 made possible by GB999. After two RI
+    // class codepoints we want to force a break. Do this by resetting the
+    // second RI's bound class to UTF8PROC_BOUNDCLASS_OTHER, to force a break
+    // after that character according to GB999 (unless of course such a break is
+    // forbidden by a different rule such as GB9).
+    if (*state == tbc && tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR)
+      *state = UTF8PROC_BOUNDCLASS_OTHER;
+    // Special support for GB10. Fold any EXTEND codepoints into the previous
+    // boundclass if we're dealing with an emoji base boundclass.
+    else if ((*state == UTF8PROC_BOUNDCLASS_E_BASE      ||
+              *state == UTF8PROC_BOUNDCLASS_E_BASE_GAZ) &&
+             tbc == UTF8PROC_BOUNDCLASS_EXTEND)
+      *state = UTF8PROC_BOUNDCLASS_E_BASE;
+    else
+      *state = tbc;
+  }
+  return break_permitted;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
+    utf8proc_int32_t c1, utf8proc_int32_t c2, utf8proc_int32_t *state) {
+
+  return grapheme_break_extended(utf8proc_get_property(c1)->boundclass,
+                                 utf8proc_get_property(c2)->boundclass,
+                                 state);
+}
+
+
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
+    utf8proc_int32_t c1, utf8proc_int32_t c2) {
+  return utf8proc_grapheme_break_stateful(c1, c2, NULL);
+}
+
+static utf8proc_int32_t seqindex_decode_entry(const utf8proc_uint16_t **entry)
+{
+  utf8proc_int32_t entry_cp = **entry;
+  if ((entry_cp & 0xF800) == 0xD800) {
+    *entry = *entry + 1;
+    entry_cp = ((entry_cp & 0x03FF) << 10) | (**entry & 0x03FF);
+    entry_cp += 0x10000;
+  }
+  return entry_cp;
+}
+
+static utf8proc_int32_t seqindex_decode_index(const utf8proc_uint32_t seqindex)
+{
+  const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex];
+  return seqindex_decode_entry(&entry);
+}
+
+static utf8proc_ssize_t seqindex_write_char_decomposed(utf8proc_uint16_t seqindex, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
+  utf8proc_ssize_t written = 0;
+  const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex & 0x1FFF];
+  int len = seqindex >> 13;
+  if (len >= 7) {
+    len = *entry;
+    entry++;
+  }
+  for (; len >= 0; entry++, len--) {
+    utf8proc_int32_t entry_cp = seqindex_decode_entry(&entry);
+
+    written += utf8proc_decompose_char(entry_cp, dst+written,
+      (bufsize > written) ? (bufsize - written) : 0, options,
+    last_boundclass);
+    if (written < 0) return UTF8PROC_ERROR_OVERFLOW;
+  }
+  return written;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c)
+{
+  utf8proc_int32_t cl = utf8proc_get_property(c)->lowercase_seqindex;
+  return cl != UINT16_MAX ? seqindex_decode_index(cl) : c;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c)
+{
+  utf8proc_int32_t cu = utf8proc_get_property(c)->uppercase_seqindex;
+  return cu != UINT16_MAX ? seqindex_decode_index(cu) : c;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c)
+{
+  utf8proc_int32_t cu = utf8proc_get_property(c)->titlecase_seqindex;
+  return cu != UINT16_MAX ? seqindex_decode_index(cu) : c;
+}
+
+/* return a character width analogous to wcwidth (except portable and
+   hopefully less buggy than most system wcwidth functions). */
+UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t c) {
+  return utf8proc_get_property(c)->charwidth;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t c) {
+  return utf8proc_get_property(c)->category;
+}
+
+UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t c) {
+  static const char s[][3] = {"Cn","Lu","Ll","Lt","Lm","Lo","Mn","Mc","Me","Nd","Nl","No","Pc","Pd","Ps","Pe","Pi","Pf","Po","Sm","Sc","Sk","So","Zs","Zl","Zp","Cc","Cf","Cs","Co"};
+  return s[utf8proc_category(c)];
+}
+
+#define utf8proc_decompose_lump(replacement_uc) \
+  return utf8proc_decompose_char((replacement_uc), dst, bufsize, \
+  options & ~UTF8PROC_LUMP, last_boundclass)
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
+  const utf8proc_property_t *property;
+  utf8proc_propval_t category;
+  utf8proc_int32_t hangul_sindex;
+  if (uc < 0 || uc >= 0x110000) return UTF8PROC_ERROR_NOTASSIGNED;
+  property = unsafe_get_property(uc);
+  category = property->category;
+  hangul_sindex = uc - UTF8PROC_HANGUL_SBASE;
+  if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
+    if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT) {
+      utf8proc_int32_t hangul_tindex;
+      if (bufsize >= 1) {
+        dst[0] = UTF8PROC_HANGUL_LBASE +
+          hangul_sindex / UTF8PROC_HANGUL_NCOUNT;
+        if (bufsize >= 2) dst[1] = UTF8PROC_HANGUL_VBASE +
+          (hangul_sindex % UTF8PROC_HANGUL_NCOUNT) / UTF8PROC_HANGUL_TCOUNT;
+      }
+      hangul_tindex = hangul_sindex % UTF8PROC_HANGUL_TCOUNT;
+      if (!hangul_tindex) return 2;
+      if (bufsize >= 3) dst[2] = UTF8PROC_HANGUL_TBASE + hangul_tindex;
+      return 3;
+    }
+  }
+  if (options & UTF8PROC_REJECTNA) {
+    if (!category) return UTF8PROC_ERROR_NOTASSIGNED;
+  }
+  if (options & UTF8PROC_IGNORE) {
+    if (property->ignorable) return 0;
+  }
+  if (options & UTF8PROC_LUMP) {
+    if (category == UTF8PROC_CATEGORY_ZS) utf8proc_decompose_lump(0x0020);
+    if (uc == 0x2018 || uc == 0x2019 || uc == 0x02BC || uc == 0x02C8)
+      utf8proc_decompose_lump(0x0027);
+    if (category == UTF8PROC_CATEGORY_PD || uc == 0x2212)
+      utf8proc_decompose_lump(0x002D);
+    if (uc == 0x2044 || uc == 0x2215) utf8proc_decompose_lump(0x002F);
+    if (uc == 0x2236) utf8proc_decompose_lump(0x003A);
+    if (uc == 0x2039 || uc == 0x2329 || uc == 0x3008)
+      utf8proc_decompose_lump(0x003C);
+    if (uc == 0x203A || uc == 0x232A || uc == 0x3009)
+      utf8proc_decompose_lump(0x003E);
+    if (uc == 0x2216) utf8proc_decompose_lump(0x005C);
+    if (uc == 0x02C4 || uc == 0x02C6 || uc == 0x2038 || uc == 0x2303)
+      utf8proc_decompose_lump(0x005E);
+    if (category == UTF8PROC_CATEGORY_PC || uc == 0x02CD)
+      utf8proc_decompose_lump(0x005F);
+    if (uc == 0x02CB) utf8proc_decompose_lump(0x0060);
+    if (uc == 0x2223) utf8proc_decompose_lump(0x007C);
+    if (uc == 0x223C) utf8proc_decompose_lump(0x007E);
+    if ((options & UTF8PROC_NLF2LS) && (options & UTF8PROC_NLF2PS)) {
+      if (category == UTF8PROC_CATEGORY_ZL ||
+          category == UTF8PROC_CATEGORY_ZP)
+        utf8proc_decompose_lump(0x000A);
+    }
+  }
+  if (options & UTF8PROC_STRIPMARK) {
+    if (category == UTF8PROC_CATEGORY_MN ||
+      category == UTF8PROC_CATEGORY_MC ||
+      category == UTF8PROC_CATEGORY_ME) return 0;
+  }
+  if (options & UTF8PROC_CASEFOLD) {
+    if (property->casefold_seqindex != UINT16_MAX) {
+      return seqindex_write_char_decomposed(property->casefold_seqindex, dst, bufsize, options, last_boundclass);
+    }
+  }
+  if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
+    if (property->decomp_seqindex != UINT16_MAX &&
+        (!property->decomp_type || (options & UTF8PROC_COMPAT))) {
+      return seqindex_write_char_decomposed(property->decomp_seqindex, dst, bufsize, options, last_boundclass);
+    }
+  }
+  if (options & UTF8PROC_CHARBOUND) {
+    utf8proc_bool boundary;
+    int tbc = property->boundclass;
+    boundary = grapheme_break_extended(*last_boundclass, tbc, last_boundclass);
+    if (boundary) {
+      if (bufsize >= 1) dst[0] = 0xFFFF;
+      if (bufsize >= 2) dst[1] = uc;
+      return 2;
+    }
+  }
+  if (bufsize >= 1) *dst = uc;
+  return 1;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
+  utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
+) {
+    return utf8proc_decompose_custom(str, strlen, buffer, bufsize, options, NULL, NULL);
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
+  utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options,
+  utf8proc_custom_func custom_func, void *custom_data
+) {
+  /* strlen will be ignored, if UTF8PROC_NULLTERM is set in options */
+  utf8proc_ssize_t wpos = 0;
+  if ((options & UTF8PROC_COMPOSE) && (options & UTF8PROC_DECOMPOSE))
+    return UTF8PROC_ERROR_INVALIDOPTS;
+  if ((options & UTF8PROC_STRIPMARK) &&
+      !(options & UTF8PROC_COMPOSE) && !(options & UTF8PROC_DECOMPOSE))
+    return UTF8PROC_ERROR_INVALIDOPTS;
+  {
+    utf8proc_int32_t uc;
+    utf8proc_ssize_t rpos = 0;
+    utf8proc_ssize_t decomp_result;
+    int boundclass = UTF8PROC_BOUNDCLASS_START;
+    while (1) {
+      if (options & UTF8PROC_NULLTERM) {
+        rpos += utf8proc_iterate(str + rpos, -1, &uc);
+        /* checking of return value is not necessary,
+           as 'uc' is < 0 in case of error */
+        if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
+        if (rpos < 0) return UTF8PROC_ERROR_OVERFLOW;
+        if (uc == 0) break;
+      } else {
+        if (rpos >= strlen) break;
+        rpos += utf8proc_iterate(str + rpos, strlen - rpos, &uc);
+        if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
+      }
+      if (custom_func != NULL) {
+        uc = custom_func(uc, custom_data);   /* user-specified custom mapping */
+      }
+      decomp_result = utf8proc_decompose_char(
+        uc, buffer + wpos, (bufsize > wpos) ? (bufsize - wpos) : 0, options,
+        &boundclass
+      );
+      if (decomp_result < 0) return decomp_result;
+      wpos += decomp_result;
+      /* prohibiting integer overflows due to too long strings: */
+      if (wpos < 0 ||
+          wpos > (utf8proc_ssize_t)(SSIZE_MAX/sizeof(utf8proc_int32_t)/2))
+        return UTF8PROC_ERROR_OVERFLOW;
+    }
+  }
+  if ((options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && bufsize >= wpos) {
+    utf8proc_ssize_t pos = 0;
+    while (pos < wpos-1) {
+      utf8proc_int32_t uc1, uc2;
+      const utf8proc_property_t *property1, *property2;
+      uc1 = buffer[pos];
+      uc2 = buffer[pos+1];
+      property1 = unsafe_get_property(uc1);
+      property2 = unsafe_get_property(uc2);
+      if (property1->combining_class > property2->combining_class &&
+          property2->combining_class > 0) {
+        buffer[pos] = uc2;
+        buffer[pos+1] = uc1;
+        if (pos > 0) pos--; else pos++;
+      } else {
+        pos++;
+      }
+    }
+  }
+  return wpos;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
+  /* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored */
+  if (options & (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS | UTF8PROC_STRIPCC)) {
+    utf8proc_ssize_t rpos;
+    utf8proc_ssize_t wpos = 0;
+    utf8proc_int32_t uc;
+    for (rpos = 0; rpos < length; rpos++) {
+      uc = buffer[rpos];
+      if (uc == 0x000D && rpos < length-1 && buffer[rpos+1] == 0x000A) rpos++;
+      if (uc == 0x000A || uc == 0x000D || uc == 0x0085 ||
+          ((options & UTF8PROC_STRIPCC) && (uc == 0x000B || uc == 0x000C))) {
+        if (options & UTF8PROC_NLF2LS) {
+          if (options & UTF8PROC_NLF2PS) {
+            buffer[wpos++] = 0x000A;
+          } else {
+            buffer[wpos++] = 0x2028;
+          }
+        } else {
+          if (options & UTF8PROC_NLF2PS) {
+            buffer[wpos++] = 0x2029;
+          } else {
+            buffer[wpos++] = 0x0020;
+          }
+        }
+      } else if ((options & UTF8PROC_STRIPCC) &&
+          (uc < 0x0020 || (uc >= 0x007F && uc < 0x00A0))) {
+        if (uc == 0x0009) buffer[wpos++] = 0x0020;
+      } else {
+        buffer[wpos++] = uc;
+      }
+    }
+    length = wpos;
+  }
+  if (options & UTF8PROC_COMPOSE) {
+    utf8proc_int32_t *starter = NULL;
+    utf8proc_int32_t current_char;
+    const utf8proc_property_t *starter_property = NULL, *current_property;
+    utf8proc_propval_t max_combining_class = -1;
+    utf8proc_ssize_t rpos;
+    utf8proc_ssize_t wpos = 0;
+    utf8proc_int32_t composition;
+    for (rpos = 0; rpos < length; rpos++) {
+      current_char = buffer[rpos];
+      current_property = unsafe_get_property(current_char);
+      if (starter && current_property->combining_class > max_combining_class) {
+        /* combination perhaps possible */
+        utf8proc_int32_t hangul_lindex;
+        utf8proc_int32_t hangul_sindex;
+        hangul_lindex = *starter - UTF8PROC_HANGUL_LBASE;
+        if (hangul_lindex >= 0 && hangul_lindex < UTF8PROC_HANGUL_LCOUNT) {
+          utf8proc_int32_t hangul_vindex;
+          hangul_vindex = current_char - UTF8PROC_HANGUL_VBASE;
+          if (hangul_vindex >= 0 && hangul_vindex < UTF8PROC_HANGUL_VCOUNT) {
+            *starter = UTF8PROC_HANGUL_SBASE +
+              (hangul_lindex * UTF8PROC_HANGUL_VCOUNT + hangul_vindex) *
+              UTF8PROC_HANGUL_TCOUNT;
+            starter_property = NULL;
+            continue;
+          }
+        }
+        hangul_sindex = *starter - UTF8PROC_HANGUL_SBASE;
+        if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT &&
+            (hangul_sindex % UTF8PROC_HANGUL_TCOUNT) == 0) {
+          utf8proc_int32_t hangul_tindex;
+          hangul_tindex = current_char - UTF8PROC_HANGUL_TBASE;
+          if (hangul_tindex >= 0 && hangul_tindex < UTF8PROC_HANGUL_TCOUNT) {
+            *starter += hangul_tindex;
+            starter_property = NULL;
+            continue;
+          }
+        }
+        if (!starter_property) {
+          starter_property = unsafe_get_property(*starter);
+        }
+        if (starter_property->comb_index < 0x8000 &&
+            current_property->comb_index != UINT16_MAX &&
+            current_property->comb_index >= 0x8000) {
+          int sidx = starter_property->comb_index;
+          int idx = (current_property->comb_index & 0x3FFF) - utf8proc_combinations[sidx];
+          if (idx >= 0 && idx <= utf8proc_combinations[sidx + 1] ) {
+            idx += sidx + 2;
+            if (current_property->comb_index & 0x4000) {
+              composition = (utf8proc_combinations[idx] << 16) | utf8proc_combinations[idx+1];
+            } else
+              composition = utf8proc_combinations[idx];
+
+            if (composition > 0 && (!(options & UTF8PROC_STABLE) ||
+                !(unsafe_get_property(composition)->comp_exclusion))) {
+              *starter = composition;
+              starter_property = NULL;
+              continue;
+            }
+          }
+        }
+      }
+      buffer[wpos] = current_char;
+      if (current_property->combining_class) {
+        if (current_property->combining_class > max_combining_class) {
+          max_combining_class = current_property->combining_class;
+        }
+      } else {
+        starter = buffer + wpos;
+        starter_property = NULL;
+        max_combining_class = -1;
+      }
+      wpos++;
+    }
+    length = wpos;
+  }
+  return length;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
+  /* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored
+     ASSERT: 'buffer' has one spare byte of free space at the end! */
+  length = utf8proc_normalize_utf32(buffer, length, options);
+  if (length < 0) return length;
+  {
+    utf8proc_ssize_t rpos, wpos = 0;
+    utf8proc_int32_t uc;
+    if (options & UTF8PROC_CHARBOUND) {
+        for (rpos = 0; rpos < length; rpos++) {
+            uc = buffer[rpos];
+            wpos += unsafe_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
+        }
+    } else {
+        for (rpos = 0; rpos < length; rpos++) {
+            uc = buffer[rpos];
+            wpos += utf8proc_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
+        }
+    }
+    ((utf8proc_uint8_t *)buffer)[wpos] = 0;
+    return wpos;
+  }
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
+) {
+    return utf8proc_map_custom(str, strlen, dstptr, options, NULL, NULL);
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options,
+  utf8proc_custom_func custom_func, void *custom_data
+) {
+  utf8proc_int32_t *buffer;
+  utf8proc_ssize_t result;
+  *dstptr = NULL;
+  result = utf8proc_decompose_custom(str, strlen, NULL, 0, options, custom_func, custom_data);
+  if (result < 0) return result;
+  buffer = (utf8proc_int32_t *) malloc(result * sizeof(utf8proc_int32_t) + 1);
+  if (!buffer) return UTF8PROC_ERROR_NOMEM;
+  result = utf8proc_decompose_custom(str, strlen, buffer, result, options, custom_func, custom_data);
+  if (result < 0) {
+    free(buffer);
+    return result;
+  }
+  result = utf8proc_reencode(buffer, result, options);
+  if (result < 0) {
+    free(buffer);
+    return result;
+  }
+  {
+    utf8proc_int32_t *newptr;
+    newptr = (utf8proc_int32_t *) realloc(buffer, (size_t)result+1);
+    if (newptr) buffer = newptr;
+  }
+  *dstptr = (utf8proc_uint8_t *)buffer;
+  return result;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
+  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
+    UTF8PROC_DECOMPOSE);
+  return retval;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
+  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
+    UTF8PROC_COMPOSE);
+  return retval;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
+  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
+    UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
+  return retval;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
+  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
+    UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
+  return retval;
+}
--- a/lib/utf8proc-2a2f97e1/utf8proc.h
+++ b/lib/utf8proc-2a2f97e1/utf8proc.h
@ -0,0 +1,689 @@
+/*
+ * Copyright (c) 2015 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
+ * Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+
+/**
+ * @mainpage
+ *
+ * utf8proc is a free/open-source (MIT/expat licensed) C library
+ * providing Unicode normalization, case-folding, and other operations
+ * for strings in the UTF-8 encoding, supporting Unicode version
+ * 9.0.0.  See the utf8proc home page (http://julialang.org/utf8proc/)
+ * for downloads and other information, or the source code on github
+ * (https://github.com/JuliaLang/utf8proc).
+ *
+ * For the utf8proc API documentation, see: @ref utf8proc.h
+ *
+ * The features of utf8proc include:
+ *
+ * - Transformation of strings (@ref utf8proc_map) to:
+ *    - decompose (@ref UTF8PROC_DECOMPOSE) or compose (@ref UTF8PROC_COMPOSE) Unicode combining characters (http://en.wikipedia.org/wiki/Combining_character)
+ *    - canonicalize Unicode compatibility characters (@ref UTF8PROC_COMPAT)
+ *    - strip "ignorable" (@ref UTF8PROC_IGNORE) characters, control characters (@ref UTF8PROC_STRIPCC), or combining characters such as accents (@ref UTF8PROC_STRIPMARK)
+ *    - case-folding (@ref UTF8PROC_CASEFOLD)
+ * - Unicode normalization: @ref utf8proc_NFD, @ref utf8proc_NFC, @ref utf8proc_NFKD, @ref utf8proc_NFKC
+ * - Detecting grapheme boundaries (@ref utf8proc_grapheme_break and @ref UTF8PROC_CHARBOUND)
+ * - Character-width computation: @ref utf8proc_charwidth
+ * - Classification of characters by Unicode category: @ref utf8proc_category and @ref utf8proc_category_string
+ * - Encode (@ref utf8proc_encode_char) and decode (@ref utf8proc_iterate) Unicode codepoints to/from UTF-8.
+ */
+
+/** @file */
+
+#ifndef UTF8PROC_H
+#define UTF8PROC_H
+
+/** @name API version
+ *
+ * The utf8proc API version MAJOR.MINOR.PATCH, following
+ * semantic-versioning rules (http://semver.org) based on API
+ * compatibility.
+ *
+ * This is also returned at runtime by @ref utf8proc_version; however, the
+ * runtime version may append a string like "-dev" to the version number
+ * for prerelease versions.
+ *
+ * @note The shared-library version number in the Makefile
+ *       (and CMakeLists.txt, and MANIFEST) may be different,
+ *       being based on ABI compatibility rather than API compatibility.
+ */
+/** @{ */
+/** The MAJOR version number (increased when backwards API compatibility is broken). */
+#define UTF8PROC_VERSION_MAJOR 2
+/** The MINOR version number (increased when new functionality is added in a backwards-compatible manner). */
+#define UTF8PROC_VERSION_MINOR 1
+/** The PATCH version (increased for fixes that do not change the API). */
+#define UTF8PROC_VERSION_PATCH 0
+/** @} */
+
+#include <stdlib.h>
+
+#if defined(_MSC_VER) && _MSC_VER < 1800
+// MSVC prior to 2013 lacked stdbool.h and inttypes.h
+typedef signed char utf8proc_int8_t;
+typedef unsigned char utf8proc_uint8_t;
+typedef short utf8proc_int16_t;
+typedef unsigned short utf8proc_uint16_t;
+typedef int utf8proc_int32_t;
+typedef unsigned int utf8proc_uint32_t;
+#  ifdef _WIN64
+typedef __int64 utf8proc_ssize_t;
+typedef unsigned __int64 utf8proc_size_t;
+#  else
+typedef int utf8proc_ssize_t;
+typedef unsigned int utf8proc_size_t;
+#  endif
+#  ifndef __cplusplus
+// emulate C99 bool
+typedef unsigned char utf8proc_bool;
+#    ifndef __bool_true_false_are_defined
+#      define false 0
+#      define true 1
+#      define __bool_true_false_are_defined 1
+#    endif
+#  else
+typedef bool utf8proc_bool;
+#  endif
+#else
+#  include <stddef.h>
+#  include <stdbool.h>
+#  include <inttypes.h>
+typedef int8_t utf8proc_int8_t;
+typedef uint8_t utf8proc_uint8_t;
+typedef int16_t utf8proc_int16_t;
+typedef uint16_t utf8proc_uint16_t;
+typedef int32_t utf8proc_int32_t;
+typedef uint32_t utf8proc_uint32_t;
+typedef size_t utf8proc_size_t;
+typedef ptrdiff_t utf8proc_ssize_t;
+typedef bool utf8proc_bool;
+#endif
+#include <limits.h>
+
+#define UTF8PROC_DLLEXPORT
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef SSIZE_MAX
+#define SSIZE_MAX ((size_t)SIZE_MAX/2)
+#endif
+
+#ifndef UINT16_MAX
+#  define UINT16_MAX 65535U
+#endif
+
+/**
+ * Option flags used by several functions in the library.
+ */
+typedef enum {
+  /** The given UTF-8 input is NULL terminated. */
+  UTF8PROC_NULLTERM  = (1<<0),
+  /** Unicode Versioning Stability has to be respected. */
+  UTF8PROC_STABLE    = (1<<1),
+  /** Compatibility decomposition (i.e. formatting information is lost). */
+  UTF8PROC_COMPAT    = (1<<2),
+  /** Return a result with decomposed characters. */
+  UTF8PROC_COMPOSE   = (1<<3),
+  /** Return a result with decomposed characters. */
+  UTF8PROC_DECOMPOSE = (1<<4),
+  /** Strip "default ignorable characters" such as SOFT-HYPHEN or ZERO-WIDTH-SPACE. */
+  UTF8PROC_IGNORE    = (1<<5),
+  /** Return an error, if the input contains unassigned codepoints. */
+  UTF8PROC_REJECTNA  = (1<<6),
+  /**
+   * Indicating that NLF-sequences (LF, CRLF, CR, NEL) are representing a
+   * line break, and should be converted to the codepoint for line
+   * separation (LS).
+   */
+  UTF8PROC_NLF2LS    = (1<<7),
+  /**
+   * Indicating that NLF-sequences are representing a paragraph break, and
+   * should be converted to the codepoint for paragraph separation
+   * (PS).
+   */
+  UTF8PROC_NLF2PS    = (1<<8),
+  /** Indicating that the meaning of NLF-sequences is unknown. */
+  UTF8PROC_NLF2LF    = (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS),
+  /** Strips and/or convers control characters.
+   *
+   * NLF-sequences are transformed into space, except if one of the
+   * NLF2LS/PS/LF options is given. HorizontalTab (HT) and FormFeed (FF)
+   * are treated as a NLF-sequence in this case.  All other control
+   * characters are simply removed.
+   */
+  UTF8PROC_STRIPCC   = (1<<9),
+  /**
+   * Performs unicode case folding, to be able to do a case-insensitive
+   * string comparison.
+   */
+  UTF8PROC_CASEFOLD  = (1<<10),
+  /**
+   * Inserts 0xFF bytes at the beginning of each sequence which is
+   * representing a single grapheme cluster (see UAX#29).
+   */
+  UTF8PROC_CHARBOUND = (1<<11),
+  /** Lumps certain characters together.
+   *
+   * E.g. HYPHEN U+2010 and MINUS U+2212 to ASCII "-". See lump.md for details.
+   *
+   * If NLF2LF is set, this includes a transformation of paragraph and
+   * line separators to ASCII line-feed (LF).
+   */
+  UTF8PROC_LUMP      = (1<<12),
+  /** Strips all character markings.
+   *
+   * This includes non-spacing, spacing and enclosing (i.e. accents).
+   * @note This option works only with @ref UTF8PROC_COMPOSE or
+   *       @ref UTF8PROC_DECOMPOSE
+   */
+  UTF8PROC_STRIPMARK = (1<<13),
+} utf8proc_option_t;
+
+/** @name Error codes
+ * Error codes being returned by almost all functions.
+ */
+/** @{ */
+/** Memory could not be allocated. */
+#define UTF8PROC_ERROR_NOMEM -1
+/** The given string is too long to be processed. */
+#define UTF8PROC_ERROR_OVERFLOW -2
+/** The given string is not a legal UTF-8 string. */
+#define UTF8PROC_ERROR_INVALIDUTF8 -3
+/** The @ref UTF8PROC_REJECTNA flag was set and an unassigned codepoint was found. */
+#define UTF8PROC_ERROR_NOTASSIGNED -4
+/** Invalid options have been used. */
+#define UTF8PROC_ERROR_INVALIDOPTS -5
+/** @} */
+
+/* @name Types */
+
+/** Holds the value of a property. */
+typedef utf8proc_int16_t utf8proc_propval_t;
+
+/** Struct containing information about a codepoint. */
+typedef struct utf8proc_property_struct {
+  /**
+   * Unicode category.
+   * @see utf8proc_category_t.
+   */
+  utf8proc_propval_t category;
+  utf8proc_propval_t combining_class;
+  /**
+   * Bidirectional class.
+   * @see utf8proc_bidi_class_t.
+   */
+  utf8proc_propval_t bidi_class;
+  /**
+   * @anchor Decomposition type.
+   * @see utf8proc_decomp_type_t.
+   */
+  utf8proc_propval_t decomp_type;
+  utf8proc_uint16_t decomp_seqindex;
+  utf8proc_uint16_t casefold_seqindex;
+  utf8proc_uint16_t uppercase_seqindex;
+  utf8proc_uint16_t lowercase_seqindex;
+  utf8proc_uint16_t titlecase_seqindex;
+  utf8proc_uint16_t comb_index;
+  unsigned bidi_mirrored:1;
+  unsigned comp_exclusion:1;
+  /**
+   * Can this codepoint be ignored?
+   *
+   * Used by @ref utf8proc_decompose_char when @ref UTF8PROC_IGNORE is
+   * passed as an option.
+   */
+  unsigned ignorable:1;
+  unsigned control_boundary:1;
+  /** The width of the codepoint. */
+  unsigned charwidth:2;
+  unsigned pad:2;
+  /**
+   * Boundclass.
+   * @see utf8proc_boundclass_t.
+   */
+  unsigned boundclass:8;
+} utf8proc_property_t;
+
+/** Unicode categories. */
+typedef enum {
+  UTF8PROC_CATEGORY_CN  = 0, /**< Other, not assigned */
+  UTF8PROC_CATEGORY_LU  = 1, /**< Letter, uppercase */
+  UTF8PROC_CATEGORY_LL  = 2, /**< Letter, lowercase */
+  UTF8PROC_CATEGORY_LT  = 3, /**< Letter, titlecase */
+  UTF8PROC_CATEGORY_LM  = 4, /**< Letter, modifier */
+  UTF8PROC_CATEGORY_LO  = 5, /**< Letter, other */
+  UTF8PROC_CATEGORY_MN  = 6, /**< Mark, nonspacing */
+  UTF8PROC_CATEGORY_MC  = 7, /**< Mark, spacing combining */
+  UTF8PROC_CATEGORY_ME  = 8, /**< Mark, enclosing */
+  UTF8PROC_CATEGORY_ND  = 9, /**< Number, decimal digit */
+  UTF8PROC_CATEGORY_NL = 10, /**< Number, letter */
+  UTF8PROC_CATEGORY_NO = 11, /**< Number, other */
+  UTF8PROC_CATEGORY_PC = 12, /**< Punctuation, connector */
+  UTF8PROC_CATEGORY_PD = 13, /**< Punctuation, dash */
+  UTF8PROC_CATEGORY_PS = 14, /**< Punctuation, open */
+  UTF8PROC_CATEGORY_PE = 15, /**< Punctuation, close */
+  UTF8PROC_CATEGORY_PI = 16, /**< Punctuation, initial quote */
+  UTF8PROC_CATEGORY_PF = 17, /**< Punctuation, final quote */
+  UTF8PROC_CATEGORY_PO = 18, /**< Punctuation, other */
+  UTF8PROC_CATEGORY_SM = 19, /**< Symbol, math */
+  UTF8PROC_CATEGORY_SC = 20, /**< Symbol, currency */
+  UTF8PROC_CATEGORY_SK = 21, /**< Symbol, modifier */
+  UTF8PROC_CATEGORY_SO = 22, /**< Symbol, other */
+  UTF8PROC_CATEGORY_ZS = 23, /**< Separator, space */
+  UTF8PROC_CATEGORY_ZL = 24, /**< Separator, line */
+  UTF8PROC_CATEGORY_ZP = 25, /**< Separator, paragraph */
+  UTF8PROC_CATEGORY_CC = 26, /**< Other, control */
+  UTF8PROC_CATEGORY_CF = 27, /**< Other, format */
+  UTF8PROC_CATEGORY_CS = 28, /**< Other, surrogate */
+  UTF8PROC_CATEGORY_CO = 29, /**< Other, private use */
+} utf8proc_category_t;
+
+/** Bidirectional character classes. */
+typedef enum {
+  UTF8PROC_BIDI_CLASS_L     = 1, /**< Left-to-Right */
+  UTF8PROC_BIDI_CLASS_LRE   = 2, /**< Left-to-Right Embedding */
+  UTF8PROC_BIDI_CLASS_LRO   = 3, /**< Left-to-Right Override */
+  UTF8PROC_BIDI_CLASS_R     = 4, /**< Right-to-Left */
+  UTF8PROC_BIDI_CLASS_AL    = 5, /**< Right-to-Left Arabic */
+  UTF8PROC_BIDI_CLASS_RLE   = 6, /**< Right-to-Left Embedding */
+  UTF8PROC_BIDI_CLASS_RLO   = 7, /**< Right-to-Left Override */
+  UTF8PROC_BIDI_CLASS_PDF   = 8, /**< Pop Directional Format */
+  UTF8PROC_BIDI_CLASS_EN    = 9, /**< European Number */
+  UTF8PROC_BIDI_CLASS_ES   = 10, /**< European Separator */
+  UTF8PROC_BIDI_CLASS_ET   = 11, /**< European Number Terminator */
+  UTF8PROC_BIDI_CLASS_AN   = 12, /**< Arabic Number */
+  UTF8PROC_BIDI_CLASS_CS   = 13, /**< Common Number Separator */
+  UTF8PROC_BIDI_CLASS_NSM  = 14, /**< Nonspacing Mark */
+  UTF8PROC_BIDI_CLASS_BN   = 15, /**< Boundary Neutral */
+  UTF8PROC_BIDI_CLASS_B    = 16, /**< Paragraph Separator */
+  UTF8PROC_BIDI_CLASS_S    = 17, /**< Segment Separator */
+  UTF8PROC_BIDI_CLASS_WS   = 18, /**< Whitespace */
+  UTF8PROC_BIDI_CLASS_ON   = 19, /**< Other Neutrals */
+  UTF8PROC_BIDI_CLASS_LRI  = 20, /**< Left-to-Right Isolate */
+  UTF8PROC_BIDI_CLASS_RLI  = 21, /**< Right-to-Left Isolate */
+  UTF8PROC_BIDI_CLASS_FSI  = 22, /**< First Strong Isolate */
+  UTF8PROC_BIDI_CLASS_PDI  = 23, /**< Pop Directional Isolate */
+} utf8proc_bidi_class_t;
+
+/** Decomposition type. */
+typedef enum {
+  UTF8PROC_DECOMP_TYPE_FONT      = 1, /**< Font */
+  UTF8PROC_DECOMP_TYPE_NOBREAK   = 2, /**< Nobreak */
+  UTF8PROC_DECOMP_TYPE_INITIAL   = 3, /**< Initial */
+  UTF8PROC_DECOMP_TYPE_MEDIAL    = 4, /**< Medial */
+  UTF8PROC_DECOMP_TYPE_FINAL     = 5, /**< Final */
+  UTF8PROC_DECOMP_TYPE_ISOLATED  = 6, /**< Isolated */
+  UTF8PROC_DECOMP_TYPE_CIRCLE    = 7, /**< Circle */
+  UTF8PROC_DECOMP_TYPE_SUPER     = 8, /**< Super */
+  UTF8PROC_DECOMP_TYPE_SUB       = 9, /**< Sub */
+  UTF8PROC_DECOMP_TYPE_VERTICAL = 10, /**< Vertical */
+  UTF8PROC_DECOMP_TYPE_WIDE     = 11, /**< Wide */
+  UTF8PROC_DECOMP_TYPE_NARROW   = 12, /**< Narrow */
+  UTF8PROC_DECOMP_TYPE_SMALL    = 13, /**< Small */
+  UTF8PROC_DECOMP_TYPE_SQUARE   = 14, /**< Square */
+  UTF8PROC_DECOMP_TYPE_FRACTION = 15, /**< Fraction */
+  UTF8PROC_DECOMP_TYPE_COMPAT   = 16, /**< Compat */
+} utf8proc_decomp_type_t;
+
+/** Boundclass property. (TR29) */
+typedef enum {
+  UTF8PROC_BOUNDCLASS_START              =  0, /**< Start */
+  UTF8PROC_BOUNDCLASS_OTHER              =  1, /**< Other */
+  UTF8PROC_BOUNDCLASS_CR                 =  2, /**< Cr */
+  UTF8PROC_BOUNDCLASS_LF                 =  3, /**< Lf */
+  UTF8PROC_BOUNDCLASS_CONTROL            =  4, /**< Control */
+  UTF8PROC_BOUNDCLASS_EXTEND             =  5, /**< Extend */
+  UTF8PROC_BOUNDCLASS_L                  =  6, /**< L */
+  UTF8PROC_BOUNDCLASS_V                  =  7, /**< V */
+  UTF8PROC_BOUNDCLASS_T                  =  8, /**< T */
+  UTF8PROC_BOUNDCLASS_LV                 =  9, /**< Lv */
+  UTF8PROC_BOUNDCLASS_LVT                = 10, /**< Lvt */
+  UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR = 11, /**< Regional indicator */
+  UTF8PROC_BOUNDCLASS_SPACINGMARK        = 12, /**< Spacingmark */
+  UTF8PROC_BOUNDCLASS_PREPEND            = 13, /**< Prepend */
+  UTF8PROC_BOUNDCLASS_ZWJ                = 14, /**< Zero Width Joiner */
+  UTF8PROC_BOUNDCLASS_E_BASE             = 15, /**< Emoji Base */
+  UTF8PROC_BOUNDCLASS_E_MODIFIER         = 16, /**< Emoji Modifier */
+  UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ     = 17, /**< Glue_After_ZWJ */
+  UTF8PROC_BOUNDCLASS_E_BASE_GAZ         = 18, /**< E_BASE + GLUE_AFTER_ZJW */
+} utf8proc_boundclass_t;
+
+/**
+ * Function pointer type passed to @ref utf8proc_map_custom and
+ * @ref utf8proc_decompose_custom, which is used to specify a user-defined
+ * mapping of codepoints to be applied in conjunction with other mappings.
+ */
+typedef utf8proc_int32_t (*utf8proc_custom_func)(utf8proc_int32_t codepoint, void *data);
+
+/**
+ * Array containing the byte lengths of a UTF-8 encoded codepoint based
+ * on the first byte.
+ */
+UTF8PROC_DLLEXPORT extern const utf8proc_int8_t utf8proc_utf8class[256];
+
+/**
+ * Returns the utf8proc API version as a string MAJOR.MINOR.PATCH
+ * (http://semver.org format), possibly with a "-dev" suffix for
+ * development versions.
+ */
+UTF8PROC_DLLEXPORT const char *utf8proc_version(void);
+
+/**
+ * Returns an informative error string for the given utf8proc error code
+ * (e.g. the error codes returned by @ref utf8proc_map).
+ */
+UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode);
+
+/**
+ * Reads a single codepoint from the UTF-8 sequence being pointed to by `str`.
+ * The maximum number of bytes read is `strlen`, unless `strlen` is
+ * negative (in which case up to 4 bytes are read).
+ *
+ * If a valid codepoint could be read, it is stored in the variable
+ * pointed to by `codepoint_ref`, otherwise that variable will be set to -1.
+ * In case of success, the number of bytes read is returned; otherwise, a
+ * negative error code is returned.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *codepoint_ref);
+
+/**
+ * Check if a codepoint is valid (regardless of whether it has been
+ * assigned a value by the current Unicode standard).
+ *
+ * @return 1 if the given `codepoint` is valid and otherwise return 0.
+ */
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t codepoint);
+
+/**
+ * Encodes the codepoint as an UTF-8 string in the byte array pointed
+ * to by `dst`. This array must be at least 4 bytes long.
+ *
+ * In case of success the number of bytes written is returned, and
+ * otherwise 0 is returned.
+ *
+ * This function does not check whether `codepoint` is valid Unicode.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t codepoint, utf8proc_uint8_t *dst);
+
+/**
+ * Look up the properties for a given codepoint.
+ *
+ * @param codepoint The Unicode codepoint.
+ *
+ * @returns
+ * A pointer to a (constant) struct containing information about
+ * the codepoint.
+ * @par
+ * If the codepoint is unassigned or invalid, a pointer to a special struct is
+ * returned in which `category` is 0 (@ref UTF8PROC_CATEGORY_CN).
+ */
+UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t codepoint);
+
+/** Decompose a codepoint into an array of codepoints.
+ *
+ * @param codepoint the codepoint.
+ * @param dst the destination buffer.
+ * @param bufsize the size of the destination buffer.
+ * @param options one or more of the following flags:
+ * - @ref UTF8PROC_REJECTNA  - return an error `codepoint` is unassigned
+ * - @ref UTF8PROC_IGNORE    - strip "default ignorable" codepoints
+ * - @ref UTF8PROC_CASEFOLD  - apply Unicode casefolding
+ * - @ref UTF8PROC_COMPAT    - replace certain codepoints with their
+ *                             compatibility decomposition
+ * - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
+ * - @ref UTF8PROC_LUMP      - lump certain different codepoints together
+ * - @ref UTF8PROC_STRIPMARK - remove all character marks
+ * @param last_boundclass
+ * Pointer to an integer variable containing
+ * the previous codepoint's boundary class if the @ref UTF8PROC_CHARBOUND
+ * option is used.  Otherwise, this parameter is ignored.
+ *
+ * @return
+ * In case of success, the number of codepoints written is returned; in case
+ * of an error, a negative error code is returned (@ref utf8proc_errmsg).
+ * @par
+ * If the number of written codepoints would be bigger than `bufsize`, the
+ * required buffer size is returned, while the buffer will be overwritten with
+ * undefined data.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(
+  utf8proc_int32_t codepoint, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize,
+  utf8proc_option_t options, int *last_boundclass
+);
+
+/**
+ * The same as @ref utf8proc_decompose_char, but acts on a whole UTF-8
+ * string and orders the decomposed sequences correctly.
+ *
+ * If the @ref UTF8PROC_NULLTERM flag in `options` is set, processing
+ * will be stopped, when a NULL byte is encounted, otherwise `strlen`
+ * bytes are processed.  The result (in the form of 32-bit unicode
+ * codepoints) is written into the buffer being pointed to by
+ * `buffer` (which must contain at least `bufsize` entries).  In case of
+ * success, the number of codepoints written is returned; in case of an
+ * error, a negative error code is returned (@ref utf8proc_errmsg).
+ * See @ref utf8proc_decompose_custom to supply additional transformations.
+ *
+ * If the number of written codepoints would be bigger than `bufsize`, the
+ * required buffer size is returned, while the buffer will be overwritten with
+ * undefined data.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
+  utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
+);
+
+/**
+ * The same as @ref utf8proc_decompose, but also takes a `custom_func` mapping function
+ * that is called on each codepoint in `str` before any other transformations
+ * (along with a `custom_data` pointer that is passed through to `custom_func`).
+ * The `custom_func` argument is ignored if it is `NULL`.  See also @ref utf8proc_map_custom.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
+  utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options,
+  utf8proc_custom_func custom_func, void *custom_data
+);
+
+/**
+ * Normalizes the sequence of `length` codepoints pointed to by `buffer`
+ * in-place (i.e., the result is also stored in `buffer`).
+ *
+ * @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
+ * @param length the length (in codepoints) of the buffer.
+ * @param options a bitwise or (`|`) of one or more of the following flags:
+ * - @ref UTF8PROC_NLF2LS  - convert LF, CRLF, CR and NEL into LS
+ * - @ref UTF8PROC_NLF2PS  - convert LF, CRLF, CR and NEL into PS
+ * - @ref UTF8PROC_NLF2LF  - convert LF, CRLF, CR and NEL into LF
+ * - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
+ * - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
+ *                           codepoints
+ * - @ref UTF8PROC_STABLE  - prohibit combining characters that would violate
+ *                           the unicode versioning stability
+ *
+ * @return
+ * In case of success, the length (in codepoints) of the normalized UTF-32 string is
+ * returned; otherwise, a negative error code is returned (@ref utf8proc_errmsg).
+ *
+ * @warning The entries of the array pointed to by `str` have to be in the
+ *          range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
+
+/**
+ * Reencodes the sequence of `length` codepoints pointed to by `buffer`
+ * UTF-8 data in-place (i.e., the result is also stored in `buffer`).
+ * Can optionally normalize the UTF-32 sequence prior to UTF-8 conversion.
+ *
+ * @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
+ * @param length the length (in codepoints) of the buffer.
+ * @param options a bitwise or (`|`) of one or more of the following flags:
+ * - @ref UTF8PROC_NLF2LS  - convert LF, CRLF, CR and NEL into LS
+ * - @ref UTF8PROC_NLF2PS  - convert LF, CRLF, CR and NEL into PS
+ * - @ref UTF8PROC_NLF2LF  - convert LF, CRLF, CR and NEL into LF
+ * - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
+ * - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
+ *                           codepoints
+ * - @ref UTF8PROC_STABLE  - prohibit combining characters that would violate
+ *                           the unicode versioning stability
+ * - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
+ *
+ * @return
+ * In case of success, the length (in bytes) of the resulting nul-terminated
+ * UTF-8 string is returned; otherwise, a negative error code is returned
+ * (@ref utf8proc_errmsg).
+ *
+ * @warning The amount of free space pointed to by `buffer` must
+ *          exceed the amount of the input data by one byte, and the
+ *          entries of the array pointed to by `str` have to be in the
+ *          range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
+
+/**
+ * Given a pair of consecutive codepoints, return whether a grapheme break is
+ * permitted between them (as defined by the extended grapheme clusters in UAX#29).
+ *
+ * @param state Beginning with Version 29 (Unicode 9.0.0), this algorithm requires
+ *              state to break graphemes. This state can be passed in as a pointer
+ *              in the `state` argument and should initially be set to 0. If the
+ *              state is not passed in (i.e. a null pointer is passed), UAX#29 rules
+ *              GB10/12/13 which require this state will not be applied, essentially
+ *              matching the rules in Unicode 8.0.0.
+ *
+ * @warning If the state parameter is used, `utf8proc_grapheme_break_stateful` must
+ *          be called IN ORDER on ALL potential breaks in a string.
+ */
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
+    utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2, utf8proc_int32_t *state);
+
+/**
+ * Same as @ref utf8proc_grapheme_break_stateful, except without support for the
+ * Unicode 9 additions to the algorithm. Supported for legacy reasons.
+ */
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
+    utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2);
+
+
+/**
+ * Given a codepoint `c`, return the codepoint of the corresponding
+ * lower-case character, if any; otherwise (if there is no lower-case
+ * variant, or if `c` is not a valid codepoint) return `c`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c);
+
+/**
+ * Given a codepoint `c`, return the codepoint of the corresponding
+ * upper-case character, if any; otherwise (if there is no upper-case
+ * variant, or if `c` is not a valid codepoint) return `c`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c);
+
+/**
+ * Given a codepoint `c`, return the codepoint of the corresponding
+ * title-case character, if any; otherwise (if there is no title-case
+ * variant, or if `c` is not a valid codepoint) return `c`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c);
+
+/**
+ * Given a codepoint, return a character width analogous to `wcwidth(codepoint)`,
+ * except that a width of 0 is returned for non-printable codepoints
+ * instead of -1 as in `wcwidth`.
+ *
+ * @note
+ * If you want to check for particular types of non-printable characters,
+ * (analogous to `isprint` or `iscntrl`), use @ref utf8proc_category. */
+UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t codepoint);
+
+/**
+ * Return the Unicode category for the codepoint (one of the
+ * @ref utf8proc_category_t constants.)
+ */
+UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t codepoint);
+
+/**
+ * Return the two-letter (nul-terminated) Unicode category string for
+ * the codepoint (e.g. `"Lu"` or `"Co"`).
+ */
+UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t codepoint);
+
+/**
+ * Maps the given UTF-8 string pointed to by `str` to a new UTF-8
+ * string, allocated dynamically by `malloc` and returned via `dstptr`.
+ *
+ * If the @ref UTF8PROC_NULLTERM flag in the `options` field is set,
+ * the length is determined by a NULL terminator, otherwise the
+ * parameter `strlen` is evaluated to determine the string length, but
+ * in any case the result will be NULL terminated (though it might
+ * contain NULL characters with the string if `str` contained NULL
+ * characters). Other flags in the `options` field are passed to the
+ * functions defined above, and regarded as described.  See also
+ * @ref utfproc_map_custom to supply a custom codepoint transformation.
+ *
+ * In case of success the length of the new string is returned,
+ * otherwise a negative error code is returned.
+ *
+ * @note The memory of the new UTF-8 string will have been allocated
+ * with `malloc`, and should therefore be deallocated with `free`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
+);
+
+/**
+ * Like @ref utf8proc_map, but also takes a `custom_func` mapping function
+ * that is called on each codepoint in `str` before any other transformations
+ * (along with a `custom_data` pointer that is passed through to `custom_func`).
+ * The `custom_func` argument is ignored if it is `NULL`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options,
+  utf8proc_custom_func custom_func, void *custom_data
+);
+
+/** @name Unicode normalization
+ *
+ * Returns a pointer to newly allocated memory of a NFD, NFC, NFKD or NFKC
+ * normalized version of the null-terminated string `str`.  These
+ * are shortcuts to calling @ref utf8proc_map with @ref UTF8PROC_NULLTERM
+ * combined with @ref UTF8PROC_STABLE and flags indicating the normalization.
+ */
+/** @{ */
+/** NFD normalization (@ref UTF8PROC_DECOMPOSE). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str);
+/** NFC normalization (@ref UTF8PROC_COMPOSE). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str);
+/** NFKD normalization (@ref UTF8PROC_DECOMPOSE and @ref UTF8PROC_COMPAT). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str);
+/** NFKC normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str);
+/** @} */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
--- a/lib/utf8proc-2a2f97e1/utf8proc_data.c
+++ b/lib/utf8proc-2a2f97e1/utf8proc_data.c
--- a/lib/utf8proc-2a2f97e1/utils.cmake
+++ b/lib/utf8proc-2a2f97e1/utils.cmake
@ -0,0 +1,20 @@
+
+function (disallow_intree_builds)
+  # Adapted from LLVM's toplevel CMakeLists.txt file
+  if( CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR AND NOT MSVC_IDE )
+    message(FATAL_ERROR "
+      In-source builds are not allowed. CMake would overwrite the
+      makefiles distributed with utf8proc. Please create a directory
+      and run cmake from there. Building in a subdirectory is
+      fine, e.g.:
+      
+        mkdir build
+        cd build
+        cmake ..
+      
+      This process created the file `CMakeCache.txt' and the
+      directory `CMakeFiles'. Please delete them.
+      
+      ")
+  endif()
+endfunction()
--- a/lib/utf8proc.patch
+++ b/lib/utf8proc.patch
@ -0,0 +1,23 @@
+diff --git a/lib/utf8proc-2a2f97e1/utf8proc.h b/lib/utf8proc-2a2f97e1/utf8proc.h
+index 64155a1..2fca528 100644
+--- a/lib/utf8proc-2a2f97e1/utf8proc.h
+++ b/lib/utf8proc-2a2f97e1/utf8proc.h
+@@ -120,17 +120,7 @@ typedef bool utf8proc_bool;
+ #endif
+ #include <limits.h>
+ 
+-#ifdef _WIN32
+-#  ifdef UTF8PROC_EXPORTS
+-#    define UTF8PROC_DLLEXPORT __declspec(dllexport)
+-#  else
+-#    define UTF8PROC_DLLEXPORT __declspec(dllimport)
+-#  endif
+-#elif __GNUC__ >= 4
+-#  define UTF8PROC_DLLEXPORT __attribute__ ((visibility("default")))
+-#else
+-#  define UTF8PROC_DLLEXPORT
+-#endif
+#define UTF8PROC_DLLEXPORT
+ 
+ #ifdef __cplusplus
+ extern "C" {