Merge pull request #1897 from npwoods/introduce_utf8proc

Introduced utf8proc and created wrapper code to expose a prettier API
2025-10-05 08:41:31 +03:00 · 2017-01-01 08:32:39 -05:00 · 2017-01-01 08:32:39 -05:00 · e7bc92d930
commit e7bc92d930
parent 6362060a89 d863b6e90b
38 changed files with 19848 additions and 1 deletions
--- a/3rdparty/utf8proc/.travis.yml
+++ b/3rdparty/utf8proc/.travis.yml
@ -0,0 +1,22 @@
+language: c
+compiler:
+    - gcc
+    - clang
+notifications:
+    email: false
+before_install:
+    - sudo add-apt-repository ppa:staticfloat/julia-deps -y
+    - sudo add-apt-repository ppa:staticfloat/juliareleases -y
+    - sudo apt-get update -qq -y
+    - sudo apt-get install libpcre3-dev julia fontforge -y
+script:
+    - make manifest && diff MANIFEST.new MANIFEST
+    - make check
+    - make data && diff data/utf8proc_data.c.new utf8proc_data.c
+    - make clean && git status --ignored --porcelain && test -z "$(git status --ignored --porcelain)"
+    - (mkdir build_static && cd build_static && cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON && make)
+    - (mkdir build_shared && cd build_shared && cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DBUILD_SHARED_LIBS=ON && make)
+env:
+    # use JuliaLang caching (https://github.com/staticfloat/cache.julialang.org)
+    # so that Travis builds do not depend on anyone's flaky servers but our own
+    - URLCACHE=https://cache.julialang.org/ CFLAGS="-O2 -Werror -Wmissing-prototypes"
--- a/3rdparty/utf8proc/CMakeLists.txt
+++ b/3rdparty/utf8proc/CMakeLists.txt
@ -0,0 +1,33 @@
+cmake_minimum_required (VERSION 2.8)
+
+include (utils.cmake)
+
+disallow_intree_builds()
+
+project (utf8proc C)
+
+# This is the ABI version number, which may differ from the
+# API version number (defined in utf8proc.h).
+# Be sure to also update these in Makefile and MANIFEST!
+set(SO_MAJOR 2)
+set(SO_MINOR 1)
+set(SO_PATCH 0)
+
+add_definitions (
+  -DUTF8PROC_EXPORTS
+)
+
+if (NOT MSVC)
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -std=c99 -pedantic -Wall")
+endif ()
+
+add_library (utf8proc
+  utf8proc.c
+  utf8proc.h
+)
+
+set_target_properties (utf8proc PROPERTIES
+  POSITION_INDEPENDENT_CODE ON
+  VERSION "${SO_MAJOR}.${SO_MINOR}.${SO_PATCH}"
+  SOVERSION ${SO_MAJOR}
+)
--- a/3rdparty/utf8proc/Doxyfile
+++ b/3rdparty/utf8proc/Doxyfile
--- a/3rdparty/utf8proc/LICENSE.md
+++ b/3rdparty/utf8proc/LICENSE.md
@ -0,0 +1,93 @@
+## utf8proc license ##
+
+**utf8proc** is a software package originally developed
+by Jan Behrens and the rest of the Public Software Group, who
+deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers.  Like the original utf8proc,
+whose copyright and license statements are reproduced below, all new
+work on the utf8proc library is licensed under the [MIT "expat"
+license](http://opensource.org/licenses/MIT):
+
+*Copyright &copy; 2014-2015 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.*
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+## Original utf8proc license ##
+
+*Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany*
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+## Unicode data license ##
+
+This software distribution contains derived data from a modified version of
+the Unicode data files. The following license applies to that data:
+
+**COPYRIGHT AND PERMISSION NOTICE**
+
+*Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
+under the Terms of Use in http://www.unicode.org/copyright.html.*
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of the Unicode data files and any associated documentation (the "Data
+Files") or Unicode software and any associated documentation (the
+"Software") to deal in the Data Files or Software without restriction,
+including without limitation the rights to use, copy, modify, merge,
+publish, distribute, and/or sell copies of the Data Files or Software, and
+to permit persons to whom the Data Files or Software are furnished to do
+so, provided that (a) the above copyright notice(s) and this permission
+notice appear with all copies of the Data Files or Software, (b) both the
+above copyright notice(s) and this permission notice appear in associated
+documentation, and (c) there is clear notice in each modified Data File or
+in the Software as well as in the documentation associated with the Data
+File(s) or Software that the data or software has been modified.
+
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
+INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR
+CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
+USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder shall
+not be used in advertising or otherwise to promote the sale, use or other
+dealings in these Data Files or Software without prior written
+authorization of the copyright holder.
+
+Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be
+registered in some jurisdictions. All other trademarks and registered
+trademarks mentioned herein are the property of their respective owners.
--- a/3rdparty/utf8proc/MANIFEST
+++ b/3rdparty/utf8proc/MANIFEST
@ -0,0 +1,7 @@
+include/
+include/utf8proc.h
+lib/
+lib/libutf8proc.a
+lib/libutf8proc.so -> libutf8proc.so.2.1.0
+lib/libutf8proc.so.2 -> libutf8proc.so.2.1.0
+lib/libutf8proc.so.2.1.0
--- a/3rdparty/utf8proc/Makefile
+++ b/3rdparty/utf8proc/Makefile
@ -0,0 +1,150 @@
+# libutf8proc Makefile
+
+# programs
+MAKE=make
+AR?=ar
+CC?=gcc
+INSTALL=install
+FIND=find
+
+# compiler settings
+CFLAGS ?= -O2
+PICFLAG = -fPIC
+C99FLAG = -std=c99
+WCFLAGS = -Wall -pedantic
+UCFLAGS = $(CFLAGS) $(PICFLAG) $(C99FLAG) $(WCFLAGS) -DUTF8PROC_EXPORTS
+
+# shared-library version MAJOR.MINOR.PATCH ... this may be *different*
+# from the utf8proc version number because it indicates ABI compatibility,
+# not API compatibility: MAJOR should be incremented whenever *binary*
+# compatibility is broken, even if the API is backward-compatible.
+# The API version number is defined in utf8proc.h.
+# Be sure to also update these ABI versions in MANIFEST and CMakeLists.txt!
+MAJOR=2
+MINOR=1
+PATCH=0
+
+OS := $(shell uname)
+ifeq ($(OS),Darwin) # MacOS X
+  SHLIB_EXT = dylib
+  SHLIB_VERS_EXT = $(MAJOR).dylib
+else # GNU/Linux, at least (Windows should probably use cmake)
+  SHLIB_EXT = so
+  SHLIB_VERS_EXT = so.$(MAJOR).$(MINOR).$(PATCH)
+endif
+
+# installation directories (for 'make install')
+prefix=/usr/local
+libdir=$(prefix)/lib
+includedir=$(prefix)/include
+
+# meta targets
+
+.PHONY: all clean data update manifest install
+
+all: libutf8proc.a libutf8proc.$(SHLIB_EXT)
+
+clean:
+	rm -f utf8proc.o libutf8proc.a libutf8proc.$(SHLIB_VERS_EXT) libutf8proc.$(SHLIB_EXT)
+ifneq ($(OS),Darwin)
+	rm -f libutf8proc.so.$(MAJOR)
+endif
+	rm -f test/tests.o test/normtest test/graphemetest test/printproperty test/charwidth test/valid test/iterate test/case test/custom
+	rm -rf MANIFEST.new tmp
+	$(MAKE) -C bench clean
+	$(MAKE) -C data clean
+
+data: data/utf8proc_data.c.new
+
+update: data/utf8proc_data.c.new
+	cp -f data/utf8proc_data.c.new utf8proc_data.c
+
+manifest: MANIFEST.new
+
+# real targets
+
+data/utf8proc_data.c.new: libutf8proc.$(SHLIB_EXT) data/data_generator.rb data/charwidths.jl
+	$(MAKE) -C data utf8proc_data.c.new
+
+utf8proc.o: utf8proc.h utf8proc.c utf8proc_data.c
+	$(CC) $(UCFLAGS) -c -o utf8proc.o utf8proc.c
+
+libutf8proc.a: utf8proc.o
+	rm -f libutf8proc.a
+	$(AR) rs libutf8proc.a utf8proc.o
+
+libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH): utf8proc.o
+	$(CC) $(LDFLAGS) -shared -o $@ -Wl,-soname -Wl,libutf8proc.so.$(MAJOR) utf8proc.o
+	chmod a-x $@
+
+libutf8proc.so: libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH)
+	ln -f -s libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH) $@
+	ln -f -s libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH) $@.$(MAJOR)
+
+libutf8proc.$(MAJOR).dylib: utf8proc.o
+	$(CC) -dynamiclib -o $@ $^ -install_name $(libdir)/$@ -Wl,-compatibility_version -Wl,$(MAJOR) -Wl,-current_version -Wl,$(MAJOR).$(MINOR).$(PATCH)
+
+libutf8proc.dylib: libutf8proc.$(MAJOR).dylib
+	ln -f -s libutf8proc.$(MAJOR).dylib $@
+
+install: libutf8proc.a libutf8proc.$(SHLIB_EXT) libutf8proc.$(SHLIB_VERS_EXT)
+	mkdir -m 755 -p $(DESTDIR)$(includedir)
+	$(INSTALL) -m 644 utf8proc.h $(DESTDIR)$(includedir)
+	mkdir -m 755 -p $(DESTDIR)$(libdir)
+	$(INSTALL) -m 644 libutf8proc.a $(DESTDIR)$(libdir)
+	$(INSTALL) -m 755 libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)
+	ln -f -s libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)/libutf8proc.$(SHLIB_EXT)
+ifneq ($(OS),Darwin)
+	ln -f -s libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)/libutf8proc.so.$(MAJOR)
+endif
+
+MANIFEST.new:
+	rm -rf tmp
+	$(MAKE) install prefix=/usr DESTDIR=$(PWD)/tmp
+	$(FIND) tmp/usr -mindepth 1 -type l -printf "%P -> %l\n" -or -type f -printf "%P\n" -or -type d -printf "%P/\n" | LC_ALL=C sort > $@
+	rm -rf tmp
+
+# Test programs
+
+data/NormalizationTest.txt:
+	$(MAKE) -C data NormalizationTest.txt
+
+data/GraphemeBreakTest.txt:
+	$(MAKE) -C data GraphemeBreakTest.txt
+
+test/tests.o: test/tests.c test/tests.h utf8proc.h
+	$(CC) $(UCFLAGS) -c -o test/tests.o test/tests.c
+
+test/normtest: test/normtest.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/normtest.c test/tests.o utf8proc.o -o $@
+
+test/graphemetest: test/graphemetest.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/graphemetest.c test/tests.o utf8proc.o -o $@
+
+test/printproperty: test/printproperty.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/printproperty.c test/tests.o utf8proc.o -o $@
+
+test/charwidth: test/charwidth.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/charwidth.c test/tests.o utf8proc.o -o $@
+
+test/valid: test/valid.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/valid.c test/tests.o utf8proc.o -o $@
+
+test/iterate: test/iterate.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/iterate.c test/tests.o utf8proc.o -o $@
+
+test/case: test/case.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/case.c test/tests.o utf8proc.o -o $@
+
+test/custom: test/custom.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/custom.c test/tests.o utf8proc.o -o $@
+
+check: test/normtest data/NormalizationTest.txt test/graphemetest data/GraphemeBreakTest.txt test/printproperty test/case test/custom test/charwidth test/valid test/iterate bench/bench.c bench/util.c bench/util.h utf8proc.o
+	$(MAKE) -C bench
+	test/normtest data/NormalizationTest.txt
+	test/graphemetest data/GraphemeBreakTest.txt
+	test/charwidth
+	test/valid
+	test/iterate
+	test/case
+	test/custom
--- a/3rdparty/utf8proc/NEWS.md
+++ b/3rdparty/utf8proc/NEWS.md
@ -0,0 +1,303 @@
+# utf8proc release history #
+
+## Version 2.1 ##
+
+2016-12-26:
+
+- New functions `utf8proc_map_custom` and `utf8proc_decompose_custom`
+  to allow user-supplied transformations of codepoints, in conjunction
+  with other transformations ([#89]).
+
+- New function `utf8proc_normalize_utf32` to apply normalizations
+  directly to UTF-32 data (not just UTF-8) ([#88]).
+
+- Fixed stack overflow that could occur due to incorrect definition
+  of `UINT16_MAX` with some compilers ([#84]).
+
+- Fixed conflict with `stdbool.h` in Visual Studio ([#90]).
+
+- Updated font metrics to use Unifont 9.0.04.
+
+## Version 2.0.2 ##
+
+2016-07-27:
+
+- Move `-Wmissing-prototypes` warning flag from `Makefile` to `.travis.yml`
+  since MSVC does not understand this flag and it is occasionally useful to
+  build using MSVC through the `Makefile` ([#79]).
+
+- Use a different variable name for a nested loop in `bench/bench.c`, and
+  declare it in a C89 way rather than inside the `for` to avoid "error:
+  'for' loop initial declarations are only allowed in C99 mode" ([#80]).
+
+## Version 2.0.1 ##
+
+2016-07-13:
+
+- Bug fix in `utf8proc_grapheme_break_stateful` ([#77]).
+
+- Tests now use versioned Unicode files, so they will no longer
+  break when a new version of Unicode is released ([#78]).
+
+## Version 2.0 ##
+
+2016-07-13:
+
+- Updated for Unicode 9.0 ([#70]).
+
+- New `utf8proc_grapheme_break_stateful` to handle the complicated
+  grapheme-breaking rules in Unicode 9.  The old `utf8proc_grapheme_break`
+  is still provided, but may incorrectly identify grapheme breaks
+  in some Unicode-9 sequences.
+
+- Smaller Unicode tables ([#62], [#68]).  This required changes
+  in the `utf8proc_property_t` structure, which breaks backward
+  compatibility if you access this `struct` directly.  The
+  functions in the API remain backward-compatible, however.
+
+- Buffer overrun fix ([#66]).
+
+## Version 1.3.1 ##
+
+2015-11-02:
+
+- Do not export symbol for internal function `unsafe_encode_char()` ([#55]).
+
+- Install relative symbolic links for shared libraries ([#58]).
+
+- Enable and fix compiler warnings ([#55], [#58]).
+
+- Add missing files to `make clean` ([#58]).
+
+## Version 1.3 ##
+
+2015-07-06:
+
+- Updated for Unicode 8.0 ([#45]).
+
+- New `utf8proc_tolower` and `utf8proc_toupper` functions, portable
+  replacements for `towlower` and `towupper` in the C library ([#40]).
+
+- Don't treat Unicode "non-characters" as invalid, and improved
+  validity checking in general ([#35]).
+
+- Prefix all typedefs with `utf8proc_`, e.g. `utf8proc_int32_t`,
+  to avoid collisions with other libraries ([#32]).
+
+- Rename `DLLEXPORT` to `UTF8PROC_DLLEXPORT` to prevent collisions.
+
+- Fix build breakage in the benchmark routines.
+
+- More fine-grained Makefile variables (`PICFLAG` etcetera), so that
+  compilation flags can be selectively overridden, and in particular
+  so that `CFLAGS` can be changed without accidentally eliminating
+  necessary flags like `-fPIC` and `-std=c99` ([#43]).
+
+- Updated character-width tables based on Unifont 8.0.01 ([#51]) and
+  the Unicode 8 character categories ([#47]).
+
+## Version 1.2 ##
+
+2015-03-28:
+
+- Updated for Unicode 7.0 ([#6]).
+
+- New function `utf8proc_grapheme_break(c1,c2)` that returns whether
+  there is a grapheme break between `c1` and `c2` ([#20]).
+
+- New function `utf8proc_charwidth(c)` that returns the number of
+  column-positions that should be required for `c`; essentially a
+  portable replacment for `wcwidth(c)` ([#27]).
+
+- New function `utf8proc_category(c)` that returns the Unicode
+  category of `c` (as one of the constants `UTF8PROC_CATEGORY_xx`).
+  Also, a function `utf8proc_category_string(c)` that returns the Unicode
+  category of `c` as a two-character string.
+
+- `cmake` script `CMakeLists.txt`, in addition to `Makefile`, for
+  easier compilation on Windows ([#28]).
+
+- Various `Makefile` improvements: a `make check` target to perform
+  tests ([#13]), `make install`, a rule to automate updating the Unicode
+  tables, etcetera.
+
+- The shared library is now versioned (e.g. has a soname on GNU/Linux) ([#24]).
+
+- C++/MSVC compatibility ([#17]).
+
+- Most `#defined` constants are now `enums` ([#29]).
+
+- New preprocessor constants `UTF8PROC_VERSION_MAJOR`,
+  `UTF8PROC_VERSION_MINOR`, and `UTF8PROC_VERSION_PATCH` for compile-time
+  detection of the API version.
+
+- Doxygen-formatted documentation ([#29]).
+
+- The Ruby and PostgreSQL plugins have been removed due to lack of testing ([#22]).
+
+## Version 1.1.6 ##
+
+2013-11-27:
+
+- PostgreSQL 9.2 and 9.3 compatibility (lowercase `c` language name)
+
+## Version 1.1.5 ##
+
+2009-08-20:
+
+- Use `RSTRING_PTR()` and `RSTRING_LEN()` instead of `RSTRING()->ptr` and
+  `RSTRING()->len` for ruby1.9 compatibility (and `#define` them, if not
+  existent)
+
+2009-10-02:
+
+- Patches for compatibility with Microsoft Visual Studio
+
+2009-10-08:
+
+- Fixes to make utf8proc usable in C++ programs
+
+2009-10-16:
+
+## Version 1.1.4 ##
+
+2009-06-14:
+
+- replaced C++ style comments for compatibility reasons
+- added typecasts to suppress compiler warnings
+- removed redundant source files for ruby-gemfile generation
+
+2009-08-19:
+
+- Changed copyright notice for Public Software Group e. V.
+- Minor changes in the `README` file
+
+## Version 1.1.3 ##
+
+2008-10-04:
+
+- Added a function `utf8proc_version` returning a string containing the version
+  number of the library.
+- Included a target `libutf8proc.dylib` for MacOSX.
+
+2009-05-01:
+- PostgreSQL 8.3 compatibility (use of `SET_VARSIZE` macro)
+
+## Version 1.1.2 ##
+
+2007-07-25:
+
+- Fixed a serious bug in the data file generator, which caused characters
+  being treated incorrectly, when stripping default ignorable characters or
+  calculating grapheme cluster boundaries.
+
+## Version 1.1.1 ##
+
+2007-06-25:
+
+- Added a new PostgreSQL function `unistrip`, which behaves like `unifold`,
+  but also removes all character marks (e.g. accents).
+
+2007-07-22:
+
+- Changed license from BSD to MIT style.
+- Added a new function `utf8proc_codepoint_valid` to the C library.
+- Changed compiler flags in `Makefile` from `-g -O0` to `-O2`
+- The ruby script, which was used to build the `utf8proc_data.c` file, is now
+  included in the distribution.
+
+## Version 1.0.3 ##
+
+2007-03-16:
+
+- Fixed a bug in the ruby library, which caused an error, when splitting an
+  empty string at grapheme cluster boundaries (method `String#utf8chars`).
+
+## Version 1.0.2 ##
+
+2006-09-21:
+
+- included a check in `Integer#utf8`, which raises an exception, if the given
+  code-point is invalid because of being too high (this was missing yet)
+
+2006-12-26:
+
+- added support for PostgreSQL version 8.2
+
+## Version 1.0.1 ##
+
+2006-09-20:
+
+- included a gem file for the ruby version of the library
+
+Release of version 1.0.1
+
+## Version 1.0 ##
+
+2006-09-17:
+
+- added the `LUMP` option, which lumps certain characters together (see `lump.md`) (also used for the PostgreSQL `unifold` function)
+- added the `STRIPMARK` option, which strips marking characters (or marks of composed characters)
+- deprecated ruby method `String#char_ary` in favour of `String#utf8chars`
+
+## Version 0.3 ##
+
+2006-07-18:
+
+- changed normalization from NFC to NFKC for postgresql unifold function
+
+2006-08-04:
+
+- added support to mark the beginning of a grapheme cluster with 0xFF (option: `CHARBOUND`)
+- added the ruby method `String#chars`, which is returning an array of UTF-8 encoded grapheme clusters
+- added `NLF2LF` transformation in postgresql `unifold` function
+- added the `DECOMPOSE` option, if you neither use `COMPOSE` or `DECOMPOSE`, no normalization will be performed (different from previous versions)
+- using integer constants rather than C-strings for character properties
+- fixed (hopefully) a problem with the ruby library on Mac OS X, which occurred when compiler optimization was switched on
+
+## Version 0.2 ##
+
+2006-06-05:
+
+- changed behaviour of PostgreSQL function to return NULL in case of invalid input, rather than raising an exceptional condition
+- improved efficiency of PostgreSQL function (no transformation to C string is done)
+
+2006-06-20:
+
+- added -fpic compiler flag in Makefile
+- fixed bug in the C code for the ruby library (usage of non-existent function)
+
+## Version 0.1 ##
+
+2006-06-02: initial release of version 0.1
+
+[#6]: https://github.com/JuliaLang/utf8proc/issues/6
+[#13]: https://github.com/JuliaLang/utf8proc/issues/13
+[#17]: https://github.com/JuliaLang/utf8proc/issues/17
+[#20]: https://github.com/JuliaLang/utf8proc/issues/20
+[#22]: https://github.com/JuliaLang/utf8proc/issues/22
+[#24]: https://github.com/JuliaLang/utf8proc/issues/24
+[#27]: https://github.com/JuliaLang/utf8proc/issues/27
+[#28]: https://github.com/JuliaLang/utf8proc/issues/28
+[#29]: https://github.com/JuliaLang/utf8proc/issues/29
+[#32]: https://github.com/JuliaLang/utf8proc/issues/32
+[#35]: https://github.com/JuliaLang/utf8proc/issues/35
+[#40]: https://github.com/JuliaLang/utf8proc/issues/40
+[#43]: https://github.com/JuliaLang/utf8proc/issues/43
+[#45]: https://github.com/JuliaLang/utf8proc/issues/45
+[#47]: https://github.com/JuliaLang/utf8proc/issues/47
+[#51]: https://github.com/JuliaLang/utf8proc/issues/51
+[#55]: https://github.com/JuliaLang/utf8proc/issues/55
+[#58]: https://github.com/JuliaLang/utf8proc/issues/58
+[#62]: https://github.com/JuliaLang/utf8proc/issues/62
+[#66]: https://github.com/JuliaLang/utf8proc/issues/66
+[#68]: https://github.com/JuliaLang/utf8proc/issues/68
+[#70]: https://github.com/JuliaLang/utf8proc/issues/70
+[#77]: https://github.com/JuliaLang/utf8proc/issues/77
+[#78]: https://github.com/JuliaLang/utf8proc/issues/78
+[#79]: https://github.com/JuliaLang/utf8proc/issues/79
+[#80]: https://github.com/JuliaLang/utf8proc/issues/80
+[#84]: https://github.com/JuliaLang/utf8proc/pull/84
+[#88]: https://github.com/JuliaLang/utf8proc/pull/88
+[#89]: https://github.com/JuliaLang/utf8proc/pull/89
+[#90]: https://github.com/JuliaLang/utf8proc/issues/90
--- a/3rdparty/utf8proc/README.md
+++ b/3rdparty/utf8proc/README.md
@ -0,0 +1,69 @@
+# utf8proc
+[![Travis CI Status](https://travis-ci.org/JuliaLang/utf8proc.png)](https://travis-ci.org/JuliaLang/utf8proc)
+[![AppVeyor Status](https://ci.appveyor.com/api/projects/status/aou20lfkyhj8xbwq/branch/master?svg=true)](https://ci.appveyor.com/project/tkelman/utf8proc/branch/master)
+
+
+[utf8proc](http://julialang.org/utf8proc/) is a small, clean C
+library that provides Unicode normalization, case-folding, and other
+operations for data in the [UTF-8
+encoding](http://en.wikipedia.org/wiki/UTF-8).  It was [initially
+developed](http://www.public-software-group.org/utf8proc) by Jan
+Behrens and the rest of the [Public Software
+Group](http://www.public-software-group.org/), who deserve *nearly all
+of the credit* for this package.  With the blessing of the Public
+Software Group, the [Julia developers](http://julialang.org/) have
+taken over development of utf8proc, since the original developers have
+moved to other projects.
+
+(utf8proc is used for basic Unicode
+support in the [Julia language](http://julialang.org/), and the Julia
+developers became involved because they wanted to add Unicode 7 support and other features.)
+
+(The original utf8proc package also includes Ruby and PostgreSQL plug-ins.
+We removed those from utf8proc in order to focus exclusively on the C
+library for the time being, but plan to add them back in or release them as separate packages.)
+
+The utf8proc package is licensed under the
+free/open-source [MIT "expat"
+license](http://opensource.org/licenses/MIT) (plus certain Unicode
+data governed by the similarly permissive [Unicode data
+license](http://www.unicode.org/copyright.html#Exhibit1)); please see
+the included `LICENSE.md` file for more detailed information.
+
+## Quick Start
+
+For compilation of the C library run `make`.
+
+## General Information
+
+The C library is found in this directory after successful compilation
+and is named `libutf8proc.a` (for the static library) and
+`libutf8proc.so` (for the dynamic library).
+
+The Unicode version supported is 9.0.0.
+
+For Unicode normalizations, the following options are used:
+
+* Normalization Form C:  `STABLE`, `COMPOSE`
+* Normalization Form D:  `STABLE`, `DECOMPOSE`
+* Normalization Form KC: `STABLE`, `COMPOSE`, `COMPAT`
+* Normalization Form KD: `STABLE`, `DECOMPOSE`, `COMPAT`
+
+## C Library
+
+The documentation for the C library is found in the `utf8proc.h` header file.
+`utf8proc_map` is function you will most likely be using for mapping UTF-8
+strings, unless you want to allocate memory yourself.
+
+## To Do
+
+See the Github [issues list](https://github.com/JuliaLang/utf8proc/issues).
+
+## Contact
+
+Bug reports, feature requests, and other queries can be filed at
+the [utf8proc issues page on Github](https://github.com/JuliaLang/utf8proc/issues).
+
+## See also
+
+An independent Lua translation of this library, [lua-mojibake](https://github.com/differentprogramming/lua-mojibake), is also available.
--- a/3rdparty/utf8proc/appveyor.yml
+++ b/3rdparty/utf8proc/appveyor.yml
@ -0,0 +1,42 @@
+branches:
+  only:
+    - master
+    - /release-.*/
+
+notifications:
+  - provider: Email
+    on_build_success: false
+    on_build_failure: false
+    on_build_status_changed: false
+
+build_script:
+  - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
+      https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
+      Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
+      throw "There are newer queued builds for this pull request, failing early." }
+  - mkdir msvc_static
+  - cd msvc_static
+  - cmake ..
+  - cmake --build .
+  - mkdir ..\msvc_shared
+  - cd ..\msvc_shared
+  - cmake .. -DBUILD_SHARED_LIBS=ON
+  - cmake --build .
+  - set PATH=C:\MinGW\bin;%PATH%
+  - C:\MinGW\msys\1.0\bin\sh --login -c "
+      cd /c/projects/utf8proc &&
+      mkdir mingw_static &&
+      cd mingw_static &&
+      cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -G'MSYS Makefiles' &&
+      make &&
+      mkdir ../mingw_shared &&
+      cd ../mingw_shared &&
+      cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DBUILD_SHARED_LIBS=ON -G'MSYS Makefiles' &&
+      make
+      "
+
+on_finish:
+  # Uncomment the following line for interactive debugging, which
+  # will print login data for a temporary remote session after the
+  # build. This requires an RDP version 6 client, e.g., FreeRDP.
+  #- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
--- a/3rdparty/utf8proc/bench/Makefile
+++ b/3rdparty/utf8proc/bench/Makefile
@ -0,0 +1,39 @@
+CURL=curl
+
+CC = cc
+CFLAGS = -O2 -std=c99 -pedantic -Wall
+
+all: bench
+
+LIBUTF8PROC = ../utf8proc.o
+
+bench: bench.o util.o $(LIBUTF8PROC)
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ bench.o util.o $(LIBUTF8PROC)
+
+DATAURL = https://raw.githubusercontent.com/duerst/eprun/master/benchmark
+DATAFILES = Deutsch_.txt Japanese_.txt Korean_.txt Vietnamese_.txt
+
+$(DATAFILES):
+	$(CURL) -O $(DATAURL)/$@
+
+bench.out: $(DATAFILES) bench
+	./bench -nfkc $(DATAFILES) > $@
+
+# you may need make CPPFLAGS=... LDFLAGS=... to help it find ICU
+icu: icu.o util.o
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ icu.o util.o -licuuc
+
+icu.out: $(DATAFILES) icu
+	./icu $(DATAFILES) > $@
+
+unistring: unistring.o util.o
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ unistring.o util.o -lunistring
+
+unistring.out: $(DATAFILES) unistring
+	./unistring $(DATAFILES) > $@
+
+.c.o:
+	$(CC) $(CPPFLAGS) -I.. $(CFLAGS) -c -o $@ $<
+
+clean:
+	rm -rf *.o *.txt bench *.out icu unistring
--- a/3rdparty/utf8proc/bench/bench.c
+++ b/3rdparty/utf8proc/bench/bench.c
@ -0,0 +1,56 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "utf8proc.h"
+#include "util.h"
+
+int main(int argc, char **argv)
+{
+	 int i, j;
+	 int options = 0;
+	 
+	 for (i = 1; i < argc; ++i) {
+		  if (!strcmp(argv[i], "-nfkc")) {
+			   options |= UTF8PROC_STABLE|UTF8PROC_COMPOSE|UTF8PROC_COMPAT;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfkd")) {
+			   options |= UTF8PROC_STABLE|UTF8PROC_DECOMPOSE|UTF8PROC_COMPAT;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfc")) {
+			   options |= UTF8PROC_STABLE|UTF8PROC_COMPOSE;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfd")) {
+			   options |= UTF8PROC_STABLE|UTF8PROC_DECOMPOSE;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-casefold")) {
+			   options |= UTF8PROC_CASEFOLD;
+			   continue;
+		  }
+		  if (argv[i][0] == '-') {
+			   fprintf(stderr, "unrecognized option: %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+
+		  size_t len;
+		  uint8_t *src = readfile(argv[i], &len);
+		  if (!src) {
+			   fprintf(stderr, "error reading %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+		  uint8_t *dest;
+		  mytime start = gettime();
+		  for (j = 0; j < 100; ++j) {
+			   utf8proc_map(src, len, &dest, options);
+			   free(dest);
+		  }
+		  printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
+		  free(src);
+	 }
+
+	 return EXIT_SUCCESS;
+}
--- a/3rdparty/utf8proc/bench/icu.c
+++ b/3rdparty/utf8proc/bench/icu.c
@ -0,0 +1,61 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+/* ICU4C */
+#include <unicode/utypes.h>
+#include <unicode/ustring.h>
+#include <unicode/ucnv.h>
+#include <unicode/unorm2.h>
+
+#include "util.h"
+
+int main(int argc, char **argv)
+{
+	 int i;
+
+	 UErrorCode err;
+	 UConverter *uc = ucnv_open("UTF8", &err);
+	 if (U_FAILURE(err)) return EXIT_FAILURE;
+
+	 const UNormalizer2 *NFKC = unorm2_getNFKCInstance(&err);
+	 if (U_FAILURE(err)) return EXIT_FAILURE;
+	 
+	 for (i = 1; i < argc; ++i) {
+		  if (argv[i][0] == '-') {
+			   fprintf(stderr, "unrecognized option: %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+
+		  size_t len;
+		  uint8_t *src = readfile(argv[i], &len);
+		  if (!src) {
+			   fprintf(stderr, "error reading %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+
+		  /* convert UTF8 data to ICU's UTF16 */
+		  UChar *usrc = (UChar*) malloc(2*len * sizeof(UChar));
+		  ucnv_toUChars(uc, usrc, 2*len, (char*) src, len, &err);
+		  if (U_FAILURE(err)) return EXIT_FAILURE;
+		  size_t ulen = u_strlen(usrc);
+
+		  /* ICU's insane normalization API requires you to
+			 know the size of the destination buffer in advance,
+			 or alternatively to repeatly try normalizing and
+			 double the buffer size until it succeeds.  Here, I just
+			 allocate a huge destination buffer to avoid the issue. */
+		  UChar *udest = (UChar*) malloc(10*ulen * sizeof(UChar));
+
+		  mytime start = gettime();
+		  for (int i = 0; i < 100; ++i) {
+			   unorm2_normalize(NFKC, usrc, ulen, udest, 10*ulen, &err);
+			   if (U_FAILURE(err)) return EXIT_FAILURE;
+		  }
+		  printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
+		  free(udest);
+		  free(usrc);
+		  free(src);
+	 }
+
+	 return EXIT_SUCCESS;
+}
--- a/3rdparty/utf8proc/bench/unistring.c
+++ b/3rdparty/utf8proc/bench/unistring.c
@ -0,0 +1,60 @@
+/* comparitive benchmark of GNU libunistring */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* libunistring */
+#include <unistr.h>
+#include <uninorm.h>
+
+#include "util.h"
+
+int main(int argc, char **argv)
+{
+	 int i;
+	 uninorm_t nf = UNINORM_NFKC;
+
+	 for (i = 1; i < argc; ++i) {
+		  if (!strcmp(argv[i], "-nfkc")) {
+			   nf = UNINORM_NFKC;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfkd")) {
+			   nf = UNINORM_NFKD;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfc")) {
+			   nf = UNINORM_NFC;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfd")) {
+			   nf = UNINORM_NFD;
+			   continue;
+		  }
+		  if (argv[i][0] == '-') {
+			   fprintf(stderr, "unrecognized option: %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+
+		  size_t len;
+		  uint8_t *src = readfile(argv[i], &len);
+		  if (!src) {
+			   fprintf(stderr, "error reading %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+
+		  size_t destlen;
+		  uint8_t *dest;
+		  mytime start = gettime();
+		  for (int i = 0; i < 100; ++i) {
+			   dest = u8_normalize(nf, src, len, NULL, &destlen);
+			   if (!dest) return EXIT_FAILURE;
+			   free(dest);
+		  }
+		  printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
+		  free(src);
+	 }
+
+	 return EXIT_SUCCESS;
+}
--- a/3rdparty/utf8proc/bench/util.c
+++ b/3rdparty/utf8proc/bench/util.c
@ -0,0 +1,39 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+
+#include "util.h"
+
+/* read file named FILENAME into an array of *len bytes,
+   returning NULL on error */
+uint8_t *readfile(const char *filename, size_t *len)
+{
+	 *len = 0;
+	 struct stat st;
+	 if (0 != stat(filename, &st)) return NULL;
+	 *len = st.st_size;
+	 FILE *f = fopen(filename, "r");
+	 if (!f) return NULL;
+	 uint8_t *s = (uint8_t *) malloc(sizeof(uint8_t) * *len);
+	 if (!s) return NULL;
+	 if (fread(s, 1, *len, f) != *len) {
+		  free(s);
+		  s = NULL;
+	 }
+	 fclose(f);
+	 return s;
+}
+
+mytime gettime(void) {
+	 mytime t;
+	 gettimeofday(&t, NULL);
+	 return t;
+}
+
+/* time difference in seconds */
+double elapsed(mytime t1, mytime t0)
+{
+     return (double)(t1.tv_sec - t0.tv_sec) +
+          (double)(t1.tv_usec - t0.tv_usec) * 1.0E-6;
+}
+
--- a/3rdparty/utf8proc/bench/util.h
+++ b/3rdparty/utf8proc/bench/util.h
@ -0,0 +1,22 @@
+#ifndef UTIL_H
+#define UTIL_H 1
+
+#include <inttypes.h>
+#include <sys/time.h>
+#include <time.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+uint8_t *readfile(const char *filename, size_t *len);
+
+typedef struct timeval mytime;
+mytime gettime(void);
+double elapsed(mytime t1, mytime t0);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* UTIL_H */
--- a/3rdparty/utf8proc/data/Makefile
+++ b/3rdparty/utf8proc/data/Makefile
@ -0,0 +1,66 @@
+# Unicode data generation rules.  Except for the test data files, most
+# users will not use these Makefile rules, which are primarily to re-generate
+# unicode_data.c when we get a new Unicode version or charwidth data; they
+# require ruby, fontforge, and julia to be installed.
+
+# programs
+CURL=curl
+RUBY=ruby
+PERL=perl
+MAKE=make
+JULIA=julia
+FONTFORGE=fontforge
+CURLFLAGS = --retry 5 --location
+
+.PHONY: clean
+
+.DELETE_ON_ERROR:
+
+utf8proc_data.c.new: data_generator.rb UnicodeData.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt CharWidths.txt
+	$(RUBY) data_generator.rb < UnicodeData.txt > $@
+
+# GNU Unifont version for font metric calculations:
+UNIFONT_VERSION=9.0.04
+
+unifont.ttf:
+	$(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont-$(UNIFONT_VERSION).ttf
+
+unifont_upper.ttf:
+	$(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont_upper-$(UNIFONT_VERSION).ttf
+
+%.sfd: %.ttf
+	$(FONTFORGE) -lang=ff -c "Open(\"$<\");Save(\"$@\");Quit(0);"
+
+CharWidths.txt: charwidths.jl unifont.sfd unifont_upper.sfd EastAsianWidth.txt
+	$(JULIA) charwidths.jl > $@
+
+# Unicode data version
+UNICODE_VERSION=9.0.0
+
+UnicodeData.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/UnicodeData.txt
+
+EastAsianWidth.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/EastAsianWidth.txt
+
+GraphemeBreakProperty.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/auxiliary/GraphemeBreakProperty.txt
+
+DerivedCoreProperties.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/DerivedCoreProperties.txt
+
+CompositionExclusions.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/CompositionExclusions.txt
+
+CaseFolding.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/CaseFolding.txt
+
+NormalizationTest.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/NormalizationTest.txt
+
+GraphemeBreakTest.txt:
+	$(CURL) $(CURLFLAGS) $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/auxiliary/GraphemeBreakTest.txt | $(PERL) -pe 's,÷,/,g;s,×,+,g' > $@
+
+clean:
+	rm -f UnicodeData.txt EastAsianWidth.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt NormalizationTest.txt GraphemeBreakTest.txt CharWidths.txt unifont*.ttf unifont*.sfd
+	rm -f utf8proc_data.c.new
--- a/3rdparty/utf8proc/data/charwidths.jl
+++ b/3rdparty/utf8proc/data/charwidths.jl
@ -0,0 +1,190 @@
+# Following work by @jiahao, we compute character widths using a combination of
+#   * advance widths from GNU Unifont (advance width 512 = 1 en)
+#   * UAX 11: East Asian Width
+#   * a few exceptions as needed
+# Adapted from http://nbviewer.ipython.org/gist/jiahao/07e8b08bf6d8671e9734
+#
+# Requires Julia (obviously) and FontForge.
+
+#############################################################################
+# Julia 0.3/0.4 compatibility (taken from Compat package)
+if VERSION < v"0.4.0-dev+1387"
+    typealias AbstractString String
+end
+if VERSION < v"0.4.0-dev+1419"
+    const UInt32 = Uint32
+end
+if VERSION < v"0.4.0-dev+3874"
+    Base.parse{T<:Integer}(::Type{T}, s::AbstractString) = parseint(T, s)
+end
+
+CharWidths = Dict{Int,Int}()
+
+#############################################################################
+# Use ../libutf8proc for category codes, rather than the one in Julia,
+# to minimize bootstrapping complexity when a new version of Unicode comes out.
+catcode(c) = ccall((:utf8proc_category,"../libutf8proc"), Cint, (Int32,), c)
+
+# use Base.UTF8proc module to get category codes constants, since
+# we won't change these in utf8proc.
+import Base.UTF8proc
+
+#############################################################################
+# Use a default width of 1 for all character categories that are
+# letter/symbol/number-like.  This can be overriden by Unifont or UAX 11
+# below, but provides a useful nonzero fallback for new codepoints when
+# a new Unicode version has been released but Unifont hasn't been updated yet.
+
+zerowidth = Set{Int}() # categories that may contain zero-width chars
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CN)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_MN)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_MC)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ME)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_SK)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZS)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZL)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZP)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CC)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CF)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CS)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CO)
+for c in 0x0000:0x110000
+    if catcode(c) ∉ zerowidth
+        CharWidths[c] = 1
+    end
+end
+
+#############################################################################
+# Widths from GNU Unifont
+
+#Read sfdfile for character widths
+function parsesfd(filename::AbstractString, CharWidths::Dict{Int,Int}=Dict{Int,Int}())
+    state=:seekchar
+    lineno = 0
+    codepoint = width = nothing
+    for line in readlines(open(filename))
+        lineno += 1
+        if state==:seekchar         #StartChar: nonmarkingreturn
+            if contains(line, "StartChar: ")
+                codepoint = nothing
+                width = nothing
+                state = :readdata
+            end
+        elseif state==:readdata #Encoding: 65538 -1 2, Width: 1024
+            contains(line, "Encoding:") && (codepoint = parse(Int, split(line)[3]))
+            contains(line, "Width:") && (width = parse(Int, split(line)[2]))
+            if codepoint!=nothing && width!=nothing && codepoint >= 0
+                w=div(width, 512) # 512 units to the en
+                if w > 0
+                    # only add nonzero widths, since (1) the default is zero
+                    # and (2) this circumvents some apparent bugs in Unifont
+                    # (https://savannah.gnu.org/bugs/index.php?45395)
+                    CharWidths[codepoint] = w
+                end
+                state = :seekchar
+            end
+        end
+    end
+    CharWidths
+end
+CharWidths=parsesfd("unifont.sfd", CharWidths)
+CharWidths=parsesfd("unifont_upper.sfd", CharWidths)
+
+#############################################################################
+# Widths from UAX #11: East Asian Width
+#   .. these take precedence over the Unifont width for all codepoints
+#      listed explicitly as wide/full/narrow/half-width
+
+for line in readlines(open("EastAsianWidth.txt"))
+    #Strip comments
+    line[1] == '#' && continue
+    precomment = split(line, '#')[1]
+    #Parse code point range and width code
+    tokens = split(precomment, ';')
+    length(tokens) >= 2 || continue
+    charrange = tokens[1]
+    width = strip(tokens[2])
+    #Parse code point range into Julia UnitRange
+    rangetokens = split(charrange, "..")
+    charstart = parse(UInt32, "0x"*rangetokens[1])
+    charend = parse(UInt32, "0x"*rangetokens[length(rangetokens)>1 ? 2 : 1])
+
+    #Assign widths
+    for c in charstart:charend
+        if width=="W" || width=="F" # wide or full
+            CharWidths[c]=2
+        elseif width=="Na"|| width=="H" # narrow or half
+            CharWidths[c]=1
+        end
+    end
+end
+
+#############################################################################
+# A few exceptions to the above cases, found by manual comparison
+# to other wcwidth functions and similar checks.
+
+for c in keys(CharWidths)
+    cat = catcode(c)
+
+    # make sure format control character (category Cf) have width 0,
+    # except for the Arabic characters 0x06xx (see unicode std 6.2, sec. 8.2)
+    if cat==UTF8proc.UTF8PROC_CATEGORY_CF && c ∉ [0x0601,0x0602,0x0603,0x06dd]
+        CharWidths[c]=0
+    end
+
+    # Unifont has nonzero width for a number of non-spacing combining
+    # characters, e.g. (in 7.0.06): f84,17b4,17b5,180b,180d,2d7f, and
+    # the variation selectors
+    if cat==UTF8proc.UTF8PROC_CATEGORY_MN
+        CharWidths[c]=0
+    end
+
+    # We also assign width of zero to unassigned and private-use
+    # codepoints (Unifont includes ConScript Unicode Registry PUA fonts,
+    # but since these are nonstandard it seems questionable to recognize them).
+    if cat==UTF8proc.UTF8PROC_CATEGORY_CO || cat==UTF8proc.UTF8PROC_CATEGORY_CN
+        CharWidths[c]=0
+    end
+
+    # for some reason, Unifont has width-2 glyphs for ASCII control chars
+    if cat==UTF8proc.UTF8PROC_CATEGORY_CC
+        CharWidths[c]=0
+    end
+end
+
+#By definition, should have zero width (on the same line)
+#0x002028 ' ' category: Zl name: LINE SEPARATOR/
+#0x002029 ' ' category: Zp name: PARAGRAPH SEPARATOR/
+CharWidths[0x2028]=0
+CharWidths[0x2029]=0
+
+#By definition, should be narrow = width of 1 en space
+#0x00202f ' ' category: Zs name: NARROW NO-BREAK SPACE/
+CharWidths[0x202f]=1
+
+#By definition, should be wide = width of 1 em space
+#0x002001 ' ' category: Zs name: EM QUAD/
+#0x002003 ' ' category: Zs name: EM SPACE/
+CharWidths[0x2001]=2
+CharWidths[0x2003]=2
+
+#############################################################################
+# Output (to a file or pipe) for processing by data_generator.rb
+# ... don't bother to output zero widths since that will be the default.
+
+firstc = 0x000000
+lastv = 0
+uhex(c) = uppercase(hex(c,4))
+for c in 0x0000:0x110000
+    v = get(CharWidths, c, 0)
+    if v != lastv || c == 0x110000
+        v < 4 || error("invalid charwidth $v for $c")
+        if firstc+1 < c
+            println(uhex(firstc), "..", uhex(c-1), "; ", lastv)
+        else
+            println(uhex(firstc), "; ", lastv)
+        end
+        firstc = c
+        lastv = v
+    end
+end
--- a/3rdparty/utf8proc/data/data_generator.rb
+++ b/3rdparty/utf8proc/data/data_generator.rb
@ -0,0 +1,411 @@
+#!/usr/bin/env ruby
+
+#  This file was used to generate the 'unicode_data.c' file by parsing the
+#  Unicode data file 'UnicodeData.txt' of the Unicode Character Database.
+#  It is included for informational purposes only and not intended for
+#  production use.
+
+
+#  Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a
+#  copy of this software and associated documentation files (the "Software"),
+#  to deal in the Software without restriction, including without limitation
+#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#  and/or sell copies of the Software, and to permit persons to whom the
+#  Software is furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#  DEALINGS IN THE SOFTWARE.
+
+
+#  This file contains derived data from a modified version of the
+#  Unicode data files. The following license applies to that data:
+#
+#  COPYRIGHT AND PERMISSION NOTICE
+#
+#  Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
+#  under the Terms of Use in http://www.unicode.org/copyright.html.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a
+#  copy of the Unicode data files and any associated documentation (the "Data
+#  Files") or Unicode software and any associated documentation (the
+#  "Software") to deal in the Data Files or Software without restriction,
+#  including without limitation the rights to use, copy, modify, merge,
+#  publish, distribute, and/or sell copies of the Data Files or Software, and
+#  to permit persons to whom the Data Files or Software are furnished to do
+#  so, provided that (a) the above copyright notice(s) and this permission
+#  notice appear with all copies of the Data Files or Software, (b) both the
+#  above copyright notice(s) and this permission notice appear in associated
+#  documentation, and (c) there is clear notice in each modified Data File or
+#  in the Software as well as in the documentation associated with the Data
+#  File(s) or Software that the data or software has been modified.
+#
+#  THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+#  KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+#  THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
+#  INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR
+#  CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
+#  USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+#  TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+#  PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+#
+#  Except as contained in this notice, the name of a copyright holder shall
+#  not be used in advertising or otherwise to promote the sale, use or other
+#  dealings in these Data Files or Software without prior written
+#  authorization of the copyright holder.
+
+
+$ignorable_list = File.read("DerivedCoreProperties.txt")[/# Derived Property: Default_Ignorable_Code_Point.*?# Total code points:/m]
+$ignorable = []
+$ignorable_list.each_line do |entry|
+  if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)/
+    $1.hex.upto($2.hex) { |e2| $ignorable << e2 }
+  elsif entry =~ /^[0-9A-F]+/
+    $ignorable << $&.hex
+  end
+end
+
+$grapheme_boundclass_list = File.read("GraphemeBreakProperty.txt")
+$grapheme_boundclass = Hash.new("UTF8PROC_BOUNDCLASS_OTHER")
+$grapheme_boundclass_list.each_line do |entry|
+  if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*([A-Za-z_]+)/
+    $1.hex.upto($2.hex) { |e2| $grapheme_boundclass[e2] = "UTF8PROC_BOUNDCLASS_" + $3.upcase }
+  elsif entry =~ /^([0-9A-F]+)\s*;\s*([A-Za-z_]+)/
+    $grapheme_boundclass[$1.hex] = "UTF8PROC_BOUNDCLASS_" + $2.upcase
+  end
+end
+
+$charwidth_list = File.read("CharWidths.txt")
+$charwidth = Hash.new(0)
+$charwidth_list.each_line do |entry|
+  if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*([0-9]+)/
+    $1.hex.upto($2.hex) { |e2| $charwidth[e2] = $3.to_i }
+  elsif entry =~ /^([0-9A-F]+)\s*;\s*([0-9]+)/
+    $charwidth[$1.hex] = $2.to_i
+  end
+end
+
+$exclusions = File.read("CompositionExclusions.txt")[/# \(1\) Script Specifics.*?# Total code points:/m]
+$exclusions = $exclusions.chomp.split("\n").collect { |e| e.hex }
+
+$excl_version = File.read("CompositionExclusions.txt")[/# \(2\) Post Composition Version precomposed characters.*?# Total code points:/m]
+$excl_version = $excl_version.chomp.split("\n").collect { |e| e.hex }
+
+$case_folding_string = File.open("CaseFolding.txt", :encoding => 'utf-8').read
+$case_folding = {}
+$case_folding_string.chomp.split("\n").each do |line|
+  next unless line =~ /([0-9A-F]+); [CFS]; ([0-9A-F ]+);/i
+  $case_folding[$1.hex] = $2.split(" ").collect { |e| e.hex }
+end
+
+$int_array = []
+$int_array_indicies = {}
+
+def str2c(string, prefix)
+  return "0" if string.nil?
+  return "UTF8PROC_#{prefix}_#{string.upcase}"
+end
+def pushary(array)
+  idx = $int_array_indicies[array]
+  unless idx
+    $int_array_indicies[array] = $int_array.length
+    idx = $int_array.length
+    array.each { |entry| $int_array << entry }
+  end
+  return idx
+end
+def cpary2utf16encoded(array)
+  return array.flat_map { |cp|
+      if (cp <= 0xFFFF)
+        raise "utf-16 code: #{cp}" if cp & 0b1111100000000000 == 0b1101100000000000
+        cp
+      else
+        temp = cp - 0x10000
+        [(temp >> 10) | 0b1101100000000000, (temp & 0b0000001111111111) | 0b1101110000000000]
+      end
+    }
+end
+def cpary2c(array)
+  return "UINT16_MAX" if array.nil? || array.length == 0
+  lencode = array.length - 1 #no sequence has len 0, so we encode len 1 as 0, len 2 as 1, ... 
+  array = cpary2utf16encoded(array)
+  if lencode >= 7 #we have only 3 bits for the length (which is already cutting it close. might need to change it to 2 bits in future Unicode versions)
+    array = [lencode] + array 
+    lencode = 7
+  end  
+  idx = pushary(array) 
+  raise "Array index out of bound" if idx > 0x1FFF
+  return "#{idx | (lencode << 13)}"
+end
+def singlecpmap(cp)
+  return "UINT16_MAX" if cp == nil
+  idx = pushary(cpary2utf16encoded([cp]))
+  raise "Array index out of bound" if idx > 0xFFFF
+  return "#{idx}"
+end
+
+class UnicodeChar
+  attr_accessor :code, :name, :category, :combining_class, :bidi_class,
+                :decomp_type, :decomp_mapping,
+                :bidi_mirrored,
+                :uppercase_mapping, :lowercase_mapping, :titlecase_mapping,
+                #caches:
+                :c_entry_index, :c_decomp_mapping, :c_case_folding
+  def initialize(line)
+    raise "Could not parse input." unless line =~ /^
+      ([0-9A-F]+);        # code
+      ([^;]+);            # name
+      ([A-Z]+);           # general category
+      ([0-9]+);           # canonical combining class
+      ([A-Z]+);           # bidi class
+      (<([A-Z]*)>)?       # decomposition type
+      ((\ ?[0-9A-F]+)*);  # decompomposition mapping
+      ([0-9]*);           # decimal digit
+      ([0-9]*);           # digit
+      ([^;]*);            # numeric
+      ([YN]*);            # bidi mirrored
+      ([^;]*);            # unicode 1.0 name
+      ([^;]*);            # iso comment
+      ([0-9A-F]*);        # simple uppercase mapping
+      ([0-9A-F]*);        # simple lowercase mapping
+      ([0-9A-F]*)$/ix     # simple titlecase mapping
+    @code              = $1.hex
+    @name              = $2
+    @category          = $3
+    @combining_class   = Integer($4)
+    @bidi_class        = $5
+    @decomp_type       = $7
+    @decomp_mapping    = ($8=='') ? nil :
+                         $8.split.collect { |element| element.hex }
+    @bidi_mirrored     = ($13=='Y') ? true : false
+    @uppercase_mapping = ($16=='') ? nil : $16.hex
+    @lowercase_mapping = ($17=='') ? nil : $17.hex
+    @titlecase_mapping = ($18=='') ? nil : $18.hex
+  end
+  def case_folding
+    $case_folding[code]
+  end
+  def c_entry(comb_indicies)
+    "  " <<
+    "{#{str2c category, 'CATEGORY'}, #{combining_class}, " <<
+    "#{str2c bidi_class, 'BIDI_CLASS'}, " <<
+    "#{str2c decomp_type, 'DECOMP_TYPE'}, " <<
+    "#{c_decomp_mapping}, " <<
+    "#{c_case_folding}, " <<
+    "#{singlecpmap uppercase_mapping }, " <<
+    "#{singlecpmap lowercase_mapping }, " <<
+    "#{singlecpmap titlecase_mapping }, " <<
+    "#{comb_indicies[code] ? comb_indicies[code]: 'UINT16_MAX'}, " <<
+    "#{bidi_mirrored}, " <<
+    "#{$exclusions.include?(code) or $excl_version.include?(code)}, " <<
+    "#{$ignorable.include?(code)}, " <<
+    "#{%W[Zl Zp Cc Cf].include?(category) and not [0x200C, 0x200D].include?(category)}, " <<
+    "#{$charwidth[code]}, 0, " <<
+    "#{$grapheme_boundclass[code]}},\n"
+  end
+end
+
+chars = []
+char_hash = {}
+
+while gets
+  if $_ =~ /^([0-9A-F]+);<[^;>,]+, First>;/i
+    first = $1.hex
+    gets
+    char = UnicodeChar.new($_)
+    raise "No last character of sequence found." unless
+      $_ =~ /^([0-9A-F]+);<([^;>,]+), Last>;/i
+    last = $1.hex
+    name = "<#{$2}>"
+    for i in first..last
+      char_clone = char.clone
+      char_clone.code = i
+      char_clone.name = name
+      char_hash[char_clone.code] = char_clone
+      chars << char_clone
+    end
+  else
+    char = UnicodeChar.new($_)
+    char_hash[char.code] = char
+    chars << char
+  end
+end
+
+comb1st_indicies = {}
+comb2nd_indicies = {}
+comb2nd_indicies_sorted_keys = []
+comb2nd_indicies_nonbasic = {}
+comb_array = []
+
+chars.each do |char|
+  if !char.nil? and char.decomp_type.nil? and char.decomp_mapping and
+      char.decomp_mapping.length == 2 and !char_hash[char.decomp_mapping[0]].nil? and
+      char_hash[char.decomp_mapping[0]].combining_class == 0 and
+      not $exclusions.include?(char.code)
+
+    dm0 = char.decomp_mapping[0]
+    dm1 = char.decomp_mapping[1]
+    unless comb1st_indicies[dm0]
+      comb1st_indicies[dm0] = comb1st_indicies.keys.length
+    end
+    unless comb2nd_indicies[dm1]
+      comb2nd_indicies_sorted_keys << dm1
+      comb2nd_indicies[dm1] = comb2nd_indicies.keys.length 
+    end
+    comb_array[comb1st_indicies[dm0]] ||= []
+    raise "Duplicate canonical mapping: #{char.code} #{dm0} #{dm1}" if comb_array[comb1st_indicies[dm0]][comb2nd_indicies[dm1]]
+    comb_array[comb1st_indicies[dm0]][comb2nd_indicies[dm1]] = char.code
+    
+    comb2nd_indicies_nonbasic[dm1] = true if char.code > 0xFFFF
+  end
+  char.c_decomp_mapping = cpary2c(char.decomp_mapping)
+  char.c_case_folding = cpary2c(char.case_folding)
+end 
+
+comb_indicies = {}
+cumoffset = 0
+comb1st_indicies_lastoffsets = []
+comb1st_indicies_firstoffsets = []
+comb1st_indicies.each do |dm0, index|
+  first = nil
+  last = nil
+  offset = 0
+  comb2nd_indicies_sorted_keys.each_with_index do |dm1, b|
+    if comb_array[index][b] 
+      first = offset unless first
+      last = offset
+      last += 1 if comb2nd_indicies_nonbasic[dm1]
+    end
+    offset += 1
+    offset += 1 if comb2nd_indicies_nonbasic[dm1]
+  end
+  comb1st_indicies_firstoffsets[index] = first
+  comb1st_indicies_lastoffsets[index] = last
+  raise "double index" if comb_indicies[dm0]
+  comb_indicies[dm0] = cumoffset
+  cumoffset += last - first + 1 + 2
+end
+
+offset = 0
+comb2nd_indicies_sorted_keys.each do |dm1|
+  raise "double index" if comb_indicies[dm1]
+  comb_indicies[dm1] = 0x8000 | (comb2nd_indicies[dm1] + offset)
+  raise "too large comb index" if  comb2nd_indicies[dm1] + offset > 0x4000
+  if comb2nd_indicies_nonbasic[dm1]
+    comb_indicies[dm1] = comb_indicies[dm1] | 0x4000
+    offset += 1
+  end
+end
+
+properties_indicies = {}
+properties = []
+chars.each do |char|
+  c_entry = char.c_entry(comb_indicies)
+  char.c_entry_index = properties_indicies[c_entry]
+  unless char.c_entry_index
+    properties_indicies[c_entry] = properties.length
+    char.c_entry_index = properties.length
+    properties << c_entry
+  end
+end
+
+stage1 = []
+stage2 = []
+for code in 0...0x110000
+  next unless code % 0x100 == 0
+  stage2_entry = []
+  for code2 in code...(code+0x100)
+    if char_hash[code2]
+      stage2_entry << (char_hash[code2].c_entry_index + 1)
+    else
+      stage2_entry << 0
+    end
+  end
+  old_index = stage2.index(stage2_entry)
+  if old_index
+    stage1 << (old_index * 0x100)
+  else
+    stage1 << (stage2.length * 0x100)
+    stage2 << stage2_entry
+  end
+end
+
+$stdout << "const utf8proc_uint16_t utf8proc_sequences[] = {\n  "
+i = 0
+$int_array.each do |entry|
+  i += 1
+  if i == 8
+    i = 0
+    $stdout << "\n  "
+  end
+  $stdout << entry << ", "
+end
+$stdout << "};\n\n"
+
+$stdout << "const utf8proc_uint16_t utf8proc_stage1table[] = {\n  "
+i = 0
+stage1.each do |entry|
+  i += 1
+  if i == 8
+    i = 0
+    $stdout << "\n  "
+  end
+  $stdout << entry << ", "
+end
+$stdout << "};\n\n"
+
+$stdout << "const utf8proc_uint16_t utf8proc_stage2table[] = {\n  "
+i = 0
+stage2.flatten.each do |entry|
+  i += 1
+  if i == 8
+    i = 0
+    $stdout << "\n  "
+  end
+  $stdout << entry << ", "
+end
+$stdout << "};\n\n"
+
+$stdout << "const utf8proc_property_t utf8proc_properties[] = {\n"
+$stdout << "  {0, 0, 0, 0, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,  false,false,false,false, 0, 0, UTF8PROC_BOUNDCLASS_OTHER},\n"
+properties.each { |line|
+  $stdout << line
+}
+$stdout << "};\n\n"
+
+
+
+$stdout << "const utf8proc_uint16_t utf8proc_combinations[] = {\n  "
+i = 0
+comb1st_indicies.keys.each_index do |a|
+  offset = 0
+  $stdout << comb1st_indicies_firstoffsets[a] << ", " << comb1st_indicies_lastoffsets[a] << ", "
+  comb2nd_indicies_sorted_keys.each_with_index do |dm1, b|
+    break if offset > comb1st_indicies_lastoffsets[a] 
+    if offset >= comb1st_indicies_firstoffsets[a]
+      i += 1
+      if i == 8
+        i = 0
+        $stdout << "\n  "
+      end
+      v = comb_array[a][b] ? comb_array[a][b] : 0
+      $stdout << (( v & 0xFFFF0000 ) >> 16) << ", " if comb2nd_indicies_nonbasic[dm1]
+      $stdout << (v & 0xFFFF) << ", "
+    end
+    offset += 1
+    offset += 1 if comb2nd_indicies_nonbasic[dm1]    
+  end
+  $stdout  << "\n"
+end
+$stdout << "};\n\n"
+
--- a/3rdparty/utf8proc/lump.md
+++ b/3rdparty/utf8proc/lump.md
@ -0,0 +1,27 @@
+```
+U+0020      <-- all space characters (general category Zs)
+U+0027  '   <-- left/right single quotation mark U+2018..2019,
+                modifier letter apostrophe U+02BC,
+                modifier letter vertical line U+02C8
+U+002D  -   <-- all dash characters (general category Pd),
+                minus U+2212
+U+002F  /   <-- fraction slash U+2044,
+                division slash U+2215
+U+003A  :   <-- ratio U+2236
+U+003C  <   <-- single left-pointing angle quotation mark U+2039,
+                left-pointing angle bracket U+2329,
+                left angle bracket U+3008
+U+003E  >   <-- single right-pointing angle quotation mark U+203A,
+                right-pointing angle bracket U+232A,
+                right angle bracket U+3009
+U+005C  \   <-- set minus U+2216
+U+005E  ^   <-- modifier letter up arrowhead U+02C4,
+                modifier letter circumflex accent U+02C6,
+                caret U+2038,
+                up arrowhead U+2303
+U+005F  _   <-- all connector characters (general category Pc),
+                modifier letter low macron U+02CD
+U+0060  `   <-- modifier letter grave accent U+02CB
+U+007C  |   <-- divides U+2223
+U+007E  ~   <-- tilde operator U+223C
+```
--- a/3rdparty/utf8proc/test/case.c
+++ b/3rdparty/utf8proc/test/case.c
@ -0,0 +1,50 @@
+#include "tests.h"
+#include <wctype.h>
+
+int main(int argc, char **argv)
+{
+     int error = 0, better = 0;
+     utf8proc_int32_t c;
+
+     (void) argc; /* unused */
+     (void) argv; /* unused */
+
+     /* some simple sanity tests of the character widths */
+     for (c = 0; c <= 0x110000; ++c) {
+          utf8proc_int32_t l = utf8proc_tolower(c);
+          utf8proc_int32_t u = utf8proc_toupper(c);
+
+          check(l == c || utf8proc_codepoint_valid(l), "invalid tolower");
+          check(u == c || utf8proc_codepoint_valid(u), "invalid toupper");
+
+          if (sizeof(wint_t) > 2 || c < (1<<16)) {
+               wint_t l0 = towlower(c), u0 = towupper(c);
+               
+               /* OS unicode tables may be out of date.  But if they
+                  do have a lower/uppercase mapping, hopefully it
+                  is correct? */
+               if (l0 != c && l0 != l) {
+                    fprintf(stderr, "MISMATCH %x != towlower(%x) == %x\n",
+                            l, c, l0);
+                    ++error;
+               }
+               else if (l0 != l) { /* often true for out-of-date OS unicode */
+                    ++better;
+                    /* printf("%x != towlower(%x) == %x\n", l, c, l0); */
+               }
+               if (u0 != c && u0 != u) {
+                    fprintf(stderr, "MISMATCH %x != towupper(%x) == %x\n",
+                            u, c, u0);
+                    ++error;
+               }
+               else if (u0 != u) { /* often true for out-of-date OS unicode */
+                    ++better;
+                    /* printf("%x != towupper(%x) == %x\n", u, c, u0); */
+               }
+          }
+     }
+     check(!error, "utf8proc case conversion FAILED %d tests.", error);
+     printf("More up-to-date than OS unicode tables for %d tests.\n", better);
+     printf("utf8proc case conversion tests SUCCEEDED.\n");
+     return 0;
+}
--- a/3rdparty/utf8proc/test/charwidth.c
+++ b/3rdparty/utf8proc/test/charwidth.c
@ -0,0 +1,71 @@
+#include "tests.h"
+#include <ctype.h>
+#include <wchar.h>
+
+static int my_isprint(int c) {
+     int cat = utf8proc_get_property(c)->category;
+     return (UTF8PROC_CATEGORY_LU <= cat && cat <= UTF8PROC_CATEGORY_ZS) ||
+          (c == 0x0601 || c == 0x0602 || c == 0x0603 || c == 0x06dd);
+}
+
+int main(int argc, char **argv)
+{
+     int c, error = 0, updates = 0;
+
+     (void) argc; /* unused */
+     (void) argv; /* unused */
+
+     /* some simple sanity tests of the character widths */
+     for (c = 0; c <= 0x110000; ++c) {
+          int cat = utf8proc_get_property(c)->category;
+          int w = utf8proc_charwidth(c);
+          if ((cat == UTF8PROC_CATEGORY_MN || cat == UTF8PROC_CATEGORY_ME) &&
+              w > 0) {
+               fprintf(stderr, "nonzero width %d for combining char %x\n", w, c);
+               error = 1;
+          }
+          if (w == 0 &&
+			  ((cat >= UTF8PROC_CATEGORY_LU && cat <= UTF8PROC_CATEGORY_LO) ||
+			   (cat >= UTF8PROC_CATEGORY_ND && cat <= UTF8PROC_CATEGORY_SC) ||
+			   (cat >= UTF8PROC_CATEGORY_SO && cat <= UTF8PROC_CATEGORY_ZS))) {
+               fprintf(stderr, "zero width for symbol-like char %x\n", c);
+               error = 1;
+          }
+          if (c <= 127 && ((!isprint(c) && w > 0) ||
+                           (isprint(c) && wcwidth(c) != w))) {
+               fprintf(stderr, "wcwidth %d mismatch %d for %s ASCII %x\n",
+                       wcwidth(c), w, 
+                       isprint(c) ? "printable" : "non-printable", c);
+               error = 1;
+          }
+          if (!my_isprint(c) && w > 0) {
+               fprintf(stderr, "non-printing %x had width %d\n", c, w);
+               error = 1;
+          }
+     }
+     check(!error, "utf8proc_charwidth FAILED tests.");
+
+     /* print some other information by compariing with system wcwidth */
+     printf("Mismatches with system wcwidth (not necessarily errors):\n");
+     for (c = 0; c <= 0x110000; ++c) {
+          int w = utf8proc_charwidth(c);
+          int wc = wcwidth(c);
+          if (sizeof(wchar_t) == 2 && c >= (1<<16)) continue;
+          /* lots of these errors for out-of-date system unicode tables */
+          if (wc == -1 && my_isprint(c) && w > 0) {
+			   updates += 1;
+#if 0
+               printf("  wcwidth(%x) = -1 for printable char\n", c);
+#endif
+		  }
+          if (wc == -1 && !my_isprint(c) && w > 0)
+               printf("  wcwidth(%x) = -1 for non-printable width-%d char\n", c, w);
+          if (wc >= 0 && wc != w)
+               printf("  wcwidth(%x) = %d != charwidth %d\n", c, wc, w);
+     }
+	 printf("   ... (positive widths for %d chars unknown to wcwidth) ...\n",
+			updates);
+     printf("Character-width tests SUCCEEDED.\n");
+
+     return 0;
+}
--- a/3rdparty/utf8proc/test/custom.c
+++ b/3rdparty/utf8proc/test/custom.c
@ -0,0 +1,27 @@
+#include "tests.h"
+
+static int thunk_test = 1;
+
+static utf8proc_int32_t custom(utf8proc_int32_t codepoint, void *thunk)
+{
+    check(((int *) thunk) == &thunk_test, "unexpected thunk passed");
+    if (codepoint == 'a')
+        return 'b';
+    if (codepoint == 'S')
+        return 0x00df; /* ß */
+    return codepoint;
+}
+
+int main(void)
+{
+    utf8proc_uint8_t input[] = {0x41,0x61,0x53,0x62,0xef,0xbd,0x81,0x00}; /* "AaSb\uff41" */
+    utf8proc_uint8_t correct[] = {0x61,0x62,0x73,0x73,0x62,0x61,0x00}; /* "abssba" */
+    utf8proc_uint8_t *output;
+    utf8proc_map_custom(input, 0, &output, UTF8PROC_CASEFOLD | UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_NULLTERM,
+                        custom, &thunk_test);
+    printf("mapped \"%s\" -> \"%s\"\n", (char*)input, (char*)output);
+    check(strlen((char*) output) == 6, "incorrect output length");
+    check(!memcmp(correct, output, 7), "incorrect output data");
+    free(output);
+    return 0;
+}
--- a/3rdparty/utf8proc/test/graphemetest.c
+++ b/3rdparty/utf8proc/test/graphemetest.c
@ -0,0 +1,74 @@
+#include "tests.h"
+
+int main(int argc, char **argv)
+{
+    char *buf = NULL;
+    size_t bufsize = 0;
+    FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
+    utf8proc_uint8_t src[1024];
+    int len;
+    
+    check(f != NULL, "error opening GraphemeBreakTest.txt");
+    while (getline(&buf, &bufsize, f) > 0) {
+        size_t bi = 0, si = 0;
+        lineno += 1;
+        
+        if (lineno % 100 == 0)
+            printf("checking line %zd...\n", lineno);
+        
+        if (buf[0] == '#') continue;
+        
+        while (buf[bi]) {
+            bi = skipspaces(buf, bi);
+            if (buf[bi] == '/') { /* grapheme break */
+                src[si++] = '/';
+                bi++;
+            }
+            else if (buf[bi] == '+') { /* no break */
+                bi++;
+            }
+            else if (buf[bi] == '#') { /* start of comments */
+                break;
+            }
+	    else { /* hex-encoded codepoint */
+                len = encode((char*) (src + si), buf + bi) - 1;
+                while (src[si]) ++si; /* advance to NUL termination */
+                bi += len;
+            }
+        }
+        if (si && src[si-1] == '/')
+            --si; /* no break after final grapheme */
+        src[si] = 0; /* NUL-terminate */
+        
+        if (si) {
+            utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */
+            size_t i = 0, j = 0;
+            utf8proc_ssize_t glen;
+            utf8proc_uint8_t *g; /* utf8proc_map grapheme results */
+            while (i < si) {
+                if (src[i] != '/')
+                    utf8[j++] = src[i++];
+                else
+                    i++;
+            }
+            glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND);
+            if (glen == UTF8PROC_ERROR_INVALIDUTF8) {
+                 /* the test file contains surrogate codepoints, which are only for UTF-16 */
+                 printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno);
+            }
+            else {
+                 check(glen >= 0, "utf8proc_map error = %s",
+                       utf8proc_errmsg(glen));
+                 for (i = 0; i <= glen; ++i)
+                      if (g[i] == 0xff)
+                          g[i] = '/'; /* easier-to-read output (/ is not in test strings) */
+                 check(!strcmp((char*)g, (char*)src),
+                       "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src);
+            }
+            free(g);
+        }
+    }
+    fclose(f);
+    printf("Passed tests after %zd lines!\n", lineno);
+    return 0;
+}
--- a/3rdparty/utf8proc/test/iterate.c
+++ b/3rdparty/utf8proc/test/iterate.c
@ -0,0 +1,166 @@
+#include "tests.h"
+#include <ctype.h>
+#include <wchar.h>
+
+static  int     tests;
+static  int     error;
+
+#define CHECKVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,len,__LINE__)
+#define CHECKINVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,UTF8PROC_ERROR_INVALIDUTF8,__LINE__)
+
+static void testbytes(unsigned char *buf, int len, utf8proc_ssize_t retval, int line)
+{
+    utf8proc_int32_t out[16];
+    utf8proc_ssize_t ret;
+
+    /* Make a copy to ensure that memory is left uninitialized after "len"
+     * bytes. This way, Valgrind can detect overreads.
+     */
+    unsigned char tmp[16];
+    memcpy(tmp, buf, len);
+
+    tests++;
+    if ((ret = utf8proc_iterate(tmp, len, out)) != retval) {
+        fprintf(stderr, "Failed (%d):", line);
+        for (int i = 0; i < len ; i++) {
+            fprintf(stderr, " 0x%02x", tmp[i]);
+        }
+        fprintf(stderr, " -> %zd\n", ret);
+        error++;
+    }
+}
+
+int main(int argc, char **argv)
+{
+    uint32_t byt;
+    unsigned char buf[16];
+
+    tests = error = 0;
+
+    // Check valid sequences that were considered valid erroneously before
+    buf[0] = 0xef;
+    buf[1] = 0xb7;
+    for (byt = 0x90; byt < 0xa0; byt++) {
+        CHECKVALID(2, byt, 3);
+    }
+    // Check 0xfffe and 0xffff
+    buf[1] = 0xbf;
+    CHECKVALID(2, 0xbe, 3);
+    CHECKVALID(2, 0xbf, 3);
+    // Check 0x??fffe & 0x??ffff
+    for (byt = 0x1fffe; byt < 0x110000; byt += 0x10000) {
+        buf[0] = 0xf0 | (byt >> 18);
+        buf[1] = 0x80 | ((byt >> 12) & 0x3f);
+        CHECKVALID(3, 0xbe, 4);
+        CHECKVALID(3, 0xbf, 4);
+    }
+    
+    // Continuation byte not after lead
+    for (byt = 0x80; byt < 0xc0; byt++) {
+        CHECKINVALID(0, byt, 1);
+    }
+
+    // Continuation byte not after lead
+    for (byt = 0x80; byt < 0xc0; byt++) {
+        CHECKINVALID(0, byt, 1);
+    }
+
+    // Test lead bytes
+    for (byt = 0xc0; byt <= 0xff; byt++) {
+	// Single lead byte at end of string
+        CHECKINVALID(0, byt, 1);
+        // Lead followed by non-continuation character < 0x80
+        CHECKINVALID(1, 65, 2);
+	// Lead followed by non-continuation character > 0xbf
+        CHECKINVALID(1, 0xc0, 2);
+    }
+
+    // Test overlong 2-byte
+    buf[0] = 0xc0;
+    for (byt = 0x81; byt <= 0xbf; byt++) {
+        CHECKINVALID(1, byt, 2);
+    }
+    buf[0] = 0xc1;
+    for (byt = 0x80; byt <= 0xbf; byt++) {
+        CHECKINVALID(1, byt, 2);
+    }
+
+    // Test overlong 3-byte
+    buf[0] = 0xe0;
+    buf[2] = 0x80;
+    for (byt = 0x80; byt <= 0x9f; byt++) {
+        CHECKINVALID(1, byt, 3);
+    }
+
+    // Test overlong 4-byte
+    buf[0] = 0xf0;
+    buf[2] = 0x80;
+    buf[3] = 0x80;
+    for (byt = 0x80; byt <= 0x8f; byt++) {
+        CHECKINVALID(1, byt, 4);
+    }
+
+    // Test 4-byte > 0x10ffff
+    buf[0] = 0xf4;
+    buf[2] = 0x80;
+    buf[3] = 0x80;
+    for (byt = 0x90; byt <= 0xbf; byt++) {
+        CHECKINVALID(1, byt, 4);
+    }
+    buf[1] = 0x80;
+    for (byt = 0xf5; byt <= 0xf7; byt++) {
+        CHECKINVALID(0, byt, 4);
+    }
+
+    // Test 5-byte
+    buf[4] = 0x80;
+    for (byt = 0xf8; byt <= 0xfb; byt++) {
+        CHECKINVALID(0, byt, 5);
+    }
+
+    // Test 6-byte
+    buf[5] = 0x80;
+    for (byt = 0xfc; byt <= 0xfd; byt++) {
+        CHECKINVALID(0, byt, 6);
+    }
+
+    // Test 7-byte
+    buf[6] = 0x80;
+    CHECKINVALID(0, 0xfe, 7);
+
+    // Three and above byte sequences
+    for (byt = 0xe0; byt < 0xf0; byt++) {
+        // Lead followed by only 1 continuation byte
+        CHECKINVALID(0, byt, 2);
+        // Lead ended by non-continuation character < 0x80
+        CHECKINVALID(2, 65, 3);
+        // Lead ended by non-continuation character > 0xbf
+        CHECKINVALID(2, 0xc0, 3);
+    }
+
+    // 3-byte encoded surrogate character(s)
+    buf[0] = 0xed; buf[2] = 0x80;
+    // Single surrogate
+    CHECKINVALID(1, 0xa0, 3);
+    // Trailing surrogate first
+    CHECKINVALID(1, 0xb0, 3);
+
+    // Four byte sequences
+    buf[1] = 0x80;
+    for (byt = 0xf0; byt < 0xf5; byt++) {
+        // Lead followed by only 1 continuation bytes
+        CHECKINVALID(0, byt, 2);
+        // Lead followed by only 2 continuation bytes
+        CHECKINVALID(0, byt, 3);
+        // Lead followed by non-continuation character < 0x80
+        CHECKINVALID(3, 65, 4);
+        // Lead followed by non-continuation character > 0xbf
+        CHECKINVALID(3, 0xc0, 4);
+
+    }
+
+     check(!error, "utf8proc_iterate FAILED %d tests out of %d", error, tests);
+     printf("utf8proc_iterate tests SUCCEEDED, (%d) tests passed.\n", tests);
+
+     return 0;
+}
--- a/3rdparty/utf8proc/test/normtest.c
+++ b/3rdparty/utf8proc/test/normtest.c
@ -0,0 +1,64 @@
+#include "tests.h"
+
+#define CHECK_NORM(NRM, norm, src) {                                 \
+    char *src_norm = (char*) utf8proc_ ## NRM((utf8proc_uint8_t*) src);      \
+    check(!strcmp(norm, src_norm),                                  \
+          "normalization failed for %s -> %s", src, norm);          \
+    free(src_norm);                                                 \
+}
+
+int main(int argc, char **argv)
+{
+     char *buf = NULL;
+     size_t bufsize = 0;
+     FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
+     char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024];
+
+     check(f != NULL, "error opening NormalizationTest.txt");
+     while (getline(&buf, &bufsize, f) > 0) {
+          size_t offset;
+          lineno += 1;
+
+          if (buf[0] == '@') {
+               printf("line %zd: %s", lineno, buf + 1);
+               continue;
+          }
+          else if (lineno % 1000 == 0)
+               printf("checking line %zd...\n", lineno);
+
+          if (buf[0] == '#') continue;
+
+          offset = encode(source, buf);
+          offset += encode(NFC, buf + offset);
+          offset += encode(NFD, buf + offset);
+          offset += encode(NFKC, buf + offset);
+          offset += encode(NFKD, buf + offset);
+
+          CHECK_NORM(NFC, NFC, source);
+          CHECK_NORM(NFC, NFC, NFC);
+          CHECK_NORM(NFC, NFC, NFD);
+          CHECK_NORM(NFC, NFKC, NFKC);
+          CHECK_NORM(NFC, NFKC, NFKD);
+
+          CHECK_NORM(NFD, NFD, source);
+          CHECK_NORM(NFD, NFD, NFC);
+          CHECK_NORM(NFD, NFD, NFD);
+          CHECK_NORM(NFD, NFKD, NFKC);
+          CHECK_NORM(NFD, NFKD, NFKD);
+
+          CHECK_NORM(NFKC, NFKC, source);
+          CHECK_NORM(NFKC, NFKC, NFC);
+          CHECK_NORM(NFKC, NFKC, NFD);
+          CHECK_NORM(NFKC, NFKC, NFKC);
+          CHECK_NORM(NFKC, NFKC, NFKD);
+
+          CHECK_NORM(NFKD, NFKD, source);
+          CHECK_NORM(NFKD, NFKD, NFC);
+          CHECK_NORM(NFKD, NFKD, NFD);
+          CHECK_NORM(NFKD, NFKD, NFKC);
+          CHECK_NORM(NFKD, NFKD, NFKD);
+     }
+     fclose(f);
+     printf("Passed tests after %zd lines!\n", lineno);
+     return 0;
+}
--- a/3rdparty/utf8proc/test/printproperty.c
+++ b/3rdparty/utf8proc/test/printproperty.c
@ -0,0 +1,49 @@
+/* simple test program to print out the utf8proc properties for a codepoint */
+
+#include "tests.h"
+
+int main(int argc, char **argv)
+{
+     int i;
+
+     for (i = 1; i < argc; ++i) {
+          unsigned int c;
+          if (!strcmp(argv[i], "-V")) {
+               printf("utf8proc version %s\n", utf8proc_version());
+               continue;
+          }
+          check(sscanf(argv[i],"%x",&c) == 1, "invalid hex input %s", argv[i]);
+          const utf8proc_property_t *p = utf8proc_get_property(c);
+          printf("U+%s:\n"
+                 "  category = %s\n"
+                 "  combining_class = %d\n"
+                 "  bidi_class = %d\n"
+                 "  decomp_type = %d\n"
+                 "  uppercase_mapping = %x\n"
+                 "  lowercase_mapping = %x\n"
+                 "  titlecase_mapping = %x\n"
+                 "  comb_index = %d\n"
+                 "  bidi_mirrored = %d\n"
+                 "  comp_exclusion = %d\n"
+                 "  ignorable = %d\n"
+                 "  control_boundary = %d\n"
+                 "  boundclass = %d\n"
+                 "  charwidth = %d\n",
+                 argv[i],
+                 utf8proc_category_string(c),
+                 p->combining_class,
+                 p->bidi_class,
+                 p->decomp_type,
+                 utf8proc_toupper(c),
+                 utf8proc_tolower(c),
+                 utf8proc_totitle(c),
+                 p->comb_index,
+                 p->bidi_mirrored,
+                 p->comp_exclusion,
+                 p->ignorable,
+                 p->control_boundary,
+                 p->boundclass,
+                 utf8proc_charwidth(c));
+     }
+     return 0;
+}
--- a/3rdparty/utf8proc/test/tests.c
+++ b/3rdparty/utf8proc/test/tests.c
@ -0,0 +1,46 @@
+/* Common functions for our test programs. */
+
+#include "tests.h"
+
+size_t lineno = 0;
+
+void check(int cond, const char *format, ...)
+{
+     if (!cond) {
+          va_list args;
+          fprintf(stderr, "line %zd: ", lineno);
+          va_start(args, format);
+          vfprintf(stderr, format, args);
+          va_end(args);
+          fprintf(stderr, "\n");
+          exit(1);
+     }
+}
+
+size_t skipspaces(const char *buf, size_t i)
+{
+    while (isspace(buf[i])) ++i;
+    return i;
+}
+
+/* if buf points to a sequence of codepoints encoded as hexadecimal strings,
+   separated by whitespace, and terminated by any character not in
+   [0-9a-fA-F] or whitespace, then stores the corresponding utf8 string
+   in dest, returning the number of bytes read from buf */
+size_t encode(char *dest, const char *buf)
+{
+     size_t i = 0, j, d = 0;
+     for (;;) {
+          int c;
+          i = skipspaces(buf, i);
+          for (j=i; buf[j] && strchr("0123456789abcdef", tolower(buf[j])); ++j)
+               ; /* find end of hex input */
+          if (j == i) { /* no codepoint found */
+               dest[d] = 0; /* NUL-terminate destination string */
+               return i + 1;
+          }
+          check(sscanf(buf + i, "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i);
+          i = j; /* skip to char after hex input */
+          d += utf8proc_encode_char(c, (utf8proc_uint8_t *) (dest + d));
+     }
+}
--- a/3rdparty/utf8proc/test/tests.h
+++ b/3rdparty/utf8proc/test/tests.h
@ -0,0 +1,23 @@
+/* Common functions and includes for our test programs. */
+
+/*
+ * Set feature macro to enable getline() and wcwidth().
+ *
+ * Please refer to section 2.2.1 of POSIX.1-2008:
+ * http://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_02_01_02
+ */
+#define _XOPEN_SOURCE 700
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+#include <stdarg.h>
+
+#include "../utf8proc.h"
+
+extern size_t lineno;
+
+void check(int cond, const char *format, ...);
+size_t skipspaces(const char *buf, size_t i);
+size_t encode(char *dest, const char *buf);
--- a/3rdparty/utf8proc/test/valid.c
+++ b/3rdparty/utf8proc/test/valid.c
@ -0,0 +1,41 @@
+#include "tests.h"
+#include <ctype.h>
+#include <wchar.h>
+
+int main(int argc, char **argv)
+{
+     int c, error = 0;
+
+     (void) argc; /* unused */
+     (void) argv; /* unused */
+
+     /* some simple sanity tests of  */
+     for (c = 0; c < 0xd800; c++) {
+        if (!utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%04x) -> false\n", c);
+           error++;
+        }
+     }
+     for (;c < 0xe000; c++) {
+        if (utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%04x) -> true\n", c);
+           error++;
+        }
+     }
+     for (;c < 0x110000; c++) {
+        if (!utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%06x) -> false\n", c);
+           error++;
+        }
+     }
+     for (;c < 0x110010; c++) {
+        if (utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%06x) -> true\n", c);
+           error++;
+        }
+     }
+     check(!error, "utf8proc_codepoint_valid FAILED %d tests.", error);
+     printf("Validity tests SUCCEEDED.\n");
+
+     return 0;
+}
--- a/3rdparty/utf8proc/utf8proc.c
+++ b/3rdparty/utf8proc/utf8proc.c
@ -0,0 +1,755 @@
+/* -*- mode: c; c-basic-offset: 2; tab-width: 2; indent-tabs-mode: nil -*- */
+/*
+ *  Copyright (c) 2015 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
+ *  Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a
+ *  copy of this software and associated documentation files (the "Software"),
+ *  to deal in the Software without restriction, including without limitation
+ *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ *  and/or sell copies of the Software, and to permit persons to whom the
+ *  Software is furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ *  DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ *  This library contains derived data from a modified version of the
+ *  Unicode data files.
+ *
+ *  The original data files are available at
+ *  http://www.unicode.org/Public/UNIDATA/
+ *
+ *  Please notice the copyright statement in the file "utf8proc_data.c".
+ */
+
+
+/*
+ *  File name:    utf8proc.c
+ *
+ *  Description:
+ *  Implementation of libutf8proc.
+ */
+
+
+#include "utf8proc.h"
+#include "utf8proc_data.c"
+
+
+UTF8PROC_DLLEXPORT const utf8proc_int8_t utf8proc_utf8class[256] = {
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+  3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+  4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0 };
+
+#define UTF8PROC_HANGUL_SBASE 0xAC00
+#define UTF8PROC_HANGUL_LBASE 0x1100
+#define UTF8PROC_HANGUL_VBASE 0x1161
+#define UTF8PROC_HANGUL_TBASE 0x11A7
+#define UTF8PROC_HANGUL_LCOUNT 19
+#define UTF8PROC_HANGUL_VCOUNT 21
+#define UTF8PROC_HANGUL_TCOUNT 28
+#define UTF8PROC_HANGUL_NCOUNT 588
+#define UTF8PROC_HANGUL_SCOUNT 11172
+/* END is exclusive */
+#define UTF8PROC_HANGUL_L_START  0x1100
+#define UTF8PROC_HANGUL_L_END    0x115A
+#define UTF8PROC_HANGUL_L_FILLER 0x115F
+#define UTF8PROC_HANGUL_V_START  0x1160
+#define UTF8PROC_HANGUL_V_END    0x11A3
+#define UTF8PROC_HANGUL_T_START  0x11A8
+#define UTF8PROC_HANGUL_T_END    0x11FA
+#define UTF8PROC_HANGUL_S_START  0xAC00
+#define UTF8PROC_HANGUL_S_END    0xD7A4
+
+/* Should follow semantic-versioning rules (semver.org) based on API
+   compatibility.  (Note that the shared-library version number will
+   be different, being based on ABI compatibility.): */
+#define STRINGIZEx(x) #x
+#define STRINGIZE(x) STRINGIZEx(x)
+UTF8PROC_DLLEXPORT const char *utf8proc_version(void) {
+  return STRINGIZE(UTF8PROC_VERSION_MAJOR) "." STRINGIZE(UTF8PROC_VERSION_MINOR) "." STRINGIZE(UTF8PROC_VERSION_PATCH) "";
+}
+
+UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode) {
+  switch (errcode) {
+    case UTF8PROC_ERROR_NOMEM:
+    return "Memory for processing UTF-8 data could not be allocated.";
+    case UTF8PROC_ERROR_OVERFLOW:
+    return "UTF-8 string is too long to be processed.";
+    case UTF8PROC_ERROR_INVALIDUTF8:
+    return "Invalid UTF-8 string";
+    case UTF8PROC_ERROR_NOTASSIGNED:
+    return "Unassigned Unicode code point found in UTF-8 string.";
+    case UTF8PROC_ERROR_INVALIDOPTS:
+    return "Invalid options for UTF-8 processing chosen.";
+    default:
+    return "An unknown error occurred while processing UTF-8 data.";
+  }
+}
+
+#define utf_cont(ch)  (((ch) & 0xc0) == 0x80)
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *dst
+) {
+  utf8proc_uint32_t uc;
+  const utf8proc_uint8_t *end;
+
+  *dst = -1;
+  if (!strlen) return 0;
+  end = str + ((strlen < 0) ? 4 : strlen);
+  uc = *str++;
+  if (uc < 0x80) {
+    *dst = uc;
+    return 1;
+  }
+  // Must be between 0xc2 and 0xf4 inclusive to be valid
+  if ((uc - 0xc2) > (0xf4-0xc2)) return UTF8PROC_ERROR_INVALIDUTF8;
+  if (uc < 0xe0) {         // 2-byte sequence
+     // Must have valid continuation character
+     if (str >= end || !utf_cont(*str)) return UTF8PROC_ERROR_INVALIDUTF8;
+     *dst = ((uc & 0x1f)<<6) | (*str & 0x3f);
+     return 2;
+  }
+  if (uc < 0xf0) {        // 3-byte sequence
+     if ((str + 1 >= end) || !utf_cont(*str) || !utf_cont(str[1]))
+        return UTF8PROC_ERROR_INVALIDUTF8;
+     // Check for surrogate chars
+     if (uc == 0xed && *str > 0x9f)
+         return UTF8PROC_ERROR_INVALIDUTF8;
+     uc = ((uc & 0xf)<<12) | ((*str & 0x3f)<<6) | (str[1] & 0x3f);
+     if (uc < 0x800)
+         return UTF8PROC_ERROR_INVALIDUTF8;
+     *dst = uc;
+     return 3;
+  }
+  // 4-byte sequence
+  // Must have 3 valid continuation characters
+  if ((str + 2 >= end) || !utf_cont(*str) || !utf_cont(str[1]) || !utf_cont(str[2]))
+     return UTF8PROC_ERROR_INVALIDUTF8;
+  // Make sure in correct range (0x10000 - 0x10ffff)
+  if (uc == 0xf0) {
+    if (*str < 0x90) return UTF8PROC_ERROR_INVALIDUTF8;
+  } else if (uc == 0xf4) {
+    if (*str > 0x8f) return UTF8PROC_ERROR_INVALIDUTF8;
+  }
+  *dst = ((uc & 7)<<18) | ((*str & 0x3f)<<12) | ((str[1] & 0x3f)<<6) | (str[2] & 0x3f);
+  return 4;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t uc) {
+    return (((utf8proc_uint32_t)uc)-0xd800 > 0x07ff) && ((utf8proc_uint32_t)uc < 0x110000);
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
+  if (uc < 0x00) {
+    return 0;
+  } else if (uc < 0x80) {
+    dst[0] = (utf8proc_uint8_t) uc;
+    return 1;
+  } else if (uc < 0x800) {
+    dst[0] = (utf8proc_uint8_t)(0xC0 + (uc >> 6));
+    dst[1] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+    return 2;
+  // Note: we allow encoding 0xd800-0xdfff here, so as not to change
+  // the API, however, these are actually invalid in UTF-8
+  } else if (uc < 0x10000) {
+    dst[0] = (utf8proc_uint8_t)(0xE0 + (uc >> 12));
+    dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
+    dst[2] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+    return 3;
+  } else if (uc < 0x110000) {
+    dst[0] = (utf8proc_uint8_t)(0xF0 + (uc >> 18));
+    dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 12) & 0x3F));
+    dst[2] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
+    dst[3] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+    return 4;
+  } else return 0;
+}
+
+/* internal "unsafe" version that does not check whether uc is in range */
+static utf8proc_ssize_t unsafe_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
+   if (uc < 0x00) {
+      return 0;
+   } else if (uc < 0x80) {
+      dst[0] = (utf8proc_uint8_t)uc;
+      return 1;
+   } else if (uc < 0x800) {
+      dst[0] = (utf8proc_uint8_t)(0xC0 + (uc >> 6));
+      dst[1] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+      return 2;
+   } else if (uc == 0xFFFF) {
+       dst[0] = (utf8proc_uint8_t)0xFF;
+       return 1;
+   } else if (uc == 0xFFFE) {
+       dst[0] = (utf8proc_uint8_t)0xFE;
+       return 1;
+   } else if (uc < 0x10000) {
+      dst[0] = (utf8proc_uint8_t)(0xE0 + (uc >> 12));
+      dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
+      dst[2] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+      return 3;
+   } else if (uc < 0x110000) {
+      dst[0] = (utf8proc_uint8_t)(0xF0 + (uc >> 18));
+      dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 12) & 0x3F));
+      dst[2] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
+      dst[3] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
+      return 4;
+   } else return 0;
+}
+
+/* internal "unsafe" version that does not check whether uc is in range */
+static const utf8proc_property_t *unsafe_get_property(utf8proc_int32_t uc) {
+  /* ASSERT: uc >= 0 && uc < 0x110000 */
+  return utf8proc_properties + (
+    utf8proc_stage2table[
+      utf8proc_stage1table[uc >> 8] + (uc & 0xFF)
+    ]
+  );
+}
+
+UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t uc) {
+  return uc < 0 || uc >= 0x110000 ? utf8proc_properties : unsafe_get_property(uc);
+}
+
+/* return whether there is a grapheme break between boundclasses lbc and tbc
+   (according to the definition of extended grapheme clusters)
+
+  Rule numbering refers to TR29 Version 29 (Unicode 9.0.0):
+  http://www.unicode.org/reports/tr29/tr29-29.html
+
+  CAVEATS:
+   Please note that evaluation of GB10 (grapheme breaks between emoji zwj sequences)
+   and GB 12/13 (regional indicator code points) require knowledge of previous characters
+   and are thus not handled by this function. This may result in an incorrect break before
+   an E_Modifier class codepoint and an incorrectly missing break between two
+   REGIONAL_INDICATOR class code points if such support does not exist in the caller.
+
+   See the special support in grapheme_break_extended, for required bookkeeping by the caller.
+*/
+static utf8proc_bool grapheme_break_simple(int lbc, int tbc) {
+  return
+    (lbc == UTF8PROC_BOUNDCLASS_START) ? true :       // GB1
+    (lbc == UTF8PROC_BOUNDCLASS_CR &&                 // GB3
+     tbc == UTF8PROC_BOUNDCLASS_LF) ? false :         // ---
+    (lbc >= UTF8PROC_BOUNDCLASS_CR && lbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true :  // GB4
+    (tbc >= UTF8PROC_BOUNDCLASS_CR && tbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true :  // GB5
+    (lbc == UTF8PROC_BOUNDCLASS_L &&                  // GB6
+     (tbc == UTF8PROC_BOUNDCLASS_L ||                 // ---
+      tbc == UTF8PROC_BOUNDCLASS_V ||                 // ---
+      tbc == UTF8PROC_BOUNDCLASS_LV ||                // ---
+      tbc == UTF8PROC_BOUNDCLASS_LVT)) ? false :      // ---
+    ((lbc == UTF8PROC_BOUNDCLASS_LV ||                // GB7
+      lbc == UTF8PROC_BOUNDCLASS_V) &&                // ---
+     (tbc == UTF8PROC_BOUNDCLASS_V ||                 // ---
+      tbc == UTF8PROC_BOUNDCLASS_T)) ? false :        // ---
+    ((lbc == UTF8PROC_BOUNDCLASS_LVT ||               // GB8
+      lbc == UTF8PROC_BOUNDCLASS_T) &&                // ---
+     tbc == UTF8PROC_BOUNDCLASS_T) ? false :          // ---
+    (tbc == UTF8PROC_BOUNDCLASS_EXTEND ||             // GB9
+     tbc == UTF8PROC_BOUNDCLASS_ZWJ ||                // ---
+     tbc == UTF8PROC_BOUNDCLASS_SPACINGMARK ||        // GB9a
+     lbc == UTF8PROC_BOUNDCLASS_PREPEND) ? false :    // GB9b
+    ((lbc == UTF8PROC_BOUNDCLASS_E_BASE ||            // GB10 (requires additional handling below)
+      lbc == UTF8PROC_BOUNDCLASS_E_BASE_GAZ) &&       // ----
+     tbc == UTF8PROC_BOUNDCLASS_E_MODIFIER) ? false : // ----
+    (lbc == UTF8PROC_BOUNDCLASS_ZWJ &&                         // GB11
+     (tbc == UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ ||             // ----
+      tbc == UTF8PROC_BOUNDCLASS_E_BASE_GAZ)) ? false :        // ----
+    (lbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR &&          // GB12/13 (requires additional handling below)
+     tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR) ? false :  // ----
+    true; // GB999
+}
+
+static utf8proc_bool grapheme_break_extended(int lbc, int tbc, utf8proc_int32_t *state)
+{
+  int lbc_override = lbc;
+  if (state && *state != UTF8PROC_BOUNDCLASS_START)
+    lbc_override = *state;
+  utf8proc_bool break_permitted = grapheme_break_simple(lbc_override, tbc);
+  if (state) {
+    // Special support for GB 12/13 made possible by GB999. After two RI
+    // class codepoints we want to force a break. Do this by resetting the
+    // second RI's bound class to UTF8PROC_BOUNDCLASS_OTHER, to force a break
+    // after that character according to GB999 (unless of course such a break is
+    // forbidden by a different rule such as GB9).
+    if (*state == tbc && tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR)
+      *state = UTF8PROC_BOUNDCLASS_OTHER;
+    // Special support for GB10. Fold any EXTEND codepoints into the previous
+    // boundclass if we're dealing with an emoji base boundclass.
+    else if ((*state == UTF8PROC_BOUNDCLASS_E_BASE      ||
+              *state == UTF8PROC_BOUNDCLASS_E_BASE_GAZ) &&
+             tbc == UTF8PROC_BOUNDCLASS_EXTEND)
+      *state = UTF8PROC_BOUNDCLASS_E_BASE;
+    else
+      *state = tbc;
+  }
+  return break_permitted;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
+    utf8proc_int32_t c1, utf8proc_int32_t c2, utf8proc_int32_t *state) {
+
+  return grapheme_break_extended(utf8proc_get_property(c1)->boundclass,
+                                 utf8proc_get_property(c2)->boundclass,
+                                 state);
+}
+
+
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
+    utf8proc_int32_t c1, utf8proc_int32_t c2) {
+  return utf8proc_grapheme_break_stateful(c1, c2, NULL);
+}
+
+static utf8proc_int32_t seqindex_decode_entry(const utf8proc_uint16_t **entry)
+{
+  utf8proc_int32_t entry_cp = **entry;
+  if ((entry_cp & 0xF800) == 0xD800) {
+    *entry = *entry + 1;
+    entry_cp = ((entry_cp & 0x03FF) << 10) | (**entry & 0x03FF);
+    entry_cp += 0x10000;
+  }
+  return entry_cp;
+}
+
+static utf8proc_int32_t seqindex_decode_index(const utf8proc_uint32_t seqindex)
+{
+  const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex];
+  return seqindex_decode_entry(&entry);
+}
+
+static utf8proc_ssize_t seqindex_write_char_decomposed(utf8proc_uint16_t seqindex, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
+  utf8proc_ssize_t written = 0;
+  const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex & 0x1FFF];
+  int len = seqindex >> 13;
+  if (len >= 7) {
+    len = *entry;
+    entry++;
+  }
+  for (; len >= 0; entry++, len--) {
+    utf8proc_int32_t entry_cp = seqindex_decode_entry(&entry);
+
+    written += utf8proc_decompose_char(entry_cp, dst+written,
+      (bufsize > written) ? (bufsize - written) : 0, options,
+    last_boundclass);
+    if (written < 0) return UTF8PROC_ERROR_OVERFLOW;
+  }
+  return written;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c)
+{
+  utf8proc_int32_t cl = utf8proc_get_property(c)->lowercase_seqindex;
+  return cl != UINT16_MAX ? seqindex_decode_index(cl) : c;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c)
+{
+  utf8proc_int32_t cu = utf8proc_get_property(c)->uppercase_seqindex;
+  return cu != UINT16_MAX ? seqindex_decode_index(cu) : c;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c)
+{
+  utf8proc_int32_t cu = utf8proc_get_property(c)->titlecase_seqindex;
+  return cu != UINT16_MAX ? seqindex_decode_index(cu) : c;
+}
+
+/* return a character width analogous to wcwidth (except portable and
+   hopefully less buggy than most system wcwidth functions). */
+UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t c) {
+  return utf8proc_get_property(c)->charwidth;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t c) {
+  return utf8proc_get_property(c)->category;
+}
+
+UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t c) {
+  static const char s[][3] = {"Cn","Lu","Ll","Lt","Lm","Lo","Mn","Mc","Me","Nd","Nl","No","Pc","Pd","Ps","Pe","Pi","Pf","Po","Sm","Sc","Sk","So","Zs","Zl","Zp","Cc","Cf","Cs","Co"};
+  return s[utf8proc_category(c)];
+}
+
+#define utf8proc_decompose_lump(replacement_uc) \
+  return utf8proc_decompose_char((replacement_uc), dst, bufsize, \
+  options & ~UTF8PROC_LUMP, last_boundclass)
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
+  const utf8proc_property_t *property;
+  utf8proc_propval_t category;
+  utf8proc_int32_t hangul_sindex;
+  if (uc < 0 || uc >= 0x110000) return UTF8PROC_ERROR_NOTASSIGNED;
+  property = unsafe_get_property(uc);
+  category = property->category;
+  hangul_sindex = uc - UTF8PROC_HANGUL_SBASE;
+  if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
+    if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT) {
+      utf8proc_int32_t hangul_tindex;
+      if (bufsize >= 1) {
+        dst[0] = UTF8PROC_HANGUL_LBASE +
+          hangul_sindex / UTF8PROC_HANGUL_NCOUNT;
+        if (bufsize >= 2) dst[1] = UTF8PROC_HANGUL_VBASE +
+          (hangul_sindex % UTF8PROC_HANGUL_NCOUNT) / UTF8PROC_HANGUL_TCOUNT;
+      }
+      hangul_tindex = hangul_sindex % UTF8PROC_HANGUL_TCOUNT;
+      if (!hangul_tindex) return 2;
+      if (bufsize >= 3) dst[2] = UTF8PROC_HANGUL_TBASE + hangul_tindex;
+      return 3;
+    }
+  }
+  if (options & UTF8PROC_REJECTNA) {
+    if (!category) return UTF8PROC_ERROR_NOTASSIGNED;
+  }
+  if (options & UTF8PROC_IGNORE) {
+    if (property->ignorable) return 0;
+  }
+  if (options & UTF8PROC_LUMP) {
+    if (category == UTF8PROC_CATEGORY_ZS) utf8proc_decompose_lump(0x0020);
+    if (uc == 0x2018 || uc == 0x2019 || uc == 0x02BC || uc == 0x02C8)
+      utf8proc_decompose_lump(0x0027);
+    if (category == UTF8PROC_CATEGORY_PD || uc == 0x2212)
+      utf8proc_decompose_lump(0x002D);
+    if (uc == 0x2044 || uc == 0x2215) utf8proc_decompose_lump(0x002F);
+    if (uc == 0x2236) utf8proc_decompose_lump(0x003A);
+    if (uc == 0x2039 || uc == 0x2329 || uc == 0x3008)
+      utf8proc_decompose_lump(0x003C);
+    if (uc == 0x203A || uc == 0x232A || uc == 0x3009)
+      utf8proc_decompose_lump(0x003E);
+    if (uc == 0x2216) utf8proc_decompose_lump(0x005C);
+    if (uc == 0x02C4 || uc == 0x02C6 || uc == 0x2038 || uc == 0x2303)
+      utf8proc_decompose_lump(0x005E);
+    if (category == UTF8PROC_CATEGORY_PC || uc == 0x02CD)
+      utf8proc_decompose_lump(0x005F);
+    if (uc == 0x02CB) utf8proc_decompose_lump(0x0060);
+    if (uc == 0x2223) utf8proc_decompose_lump(0x007C);
+    if (uc == 0x223C) utf8proc_decompose_lump(0x007E);
+    if ((options & UTF8PROC_NLF2LS) && (options & UTF8PROC_NLF2PS)) {
+      if (category == UTF8PROC_CATEGORY_ZL ||
+          category == UTF8PROC_CATEGORY_ZP)
+        utf8proc_decompose_lump(0x000A);
+    }
+  }
+  if (options & UTF8PROC_STRIPMARK) {
+    if (category == UTF8PROC_CATEGORY_MN ||
+      category == UTF8PROC_CATEGORY_MC ||
+      category == UTF8PROC_CATEGORY_ME) return 0;
+  }
+  if (options & UTF8PROC_CASEFOLD) {
+    if (property->casefold_seqindex != UINT16_MAX) {
+      return seqindex_write_char_decomposed(property->casefold_seqindex, dst, bufsize, options, last_boundclass);
+    }
+  }
+  if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
+    if (property->decomp_seqindex != UINT16_MAX &&
+        (!property->decomp_type || (options & UTF8PROC_COMPAT))) {
+      return seqindex_write_char_decomposed(property->decomp_seqindex, dst, bufsize, options, last_boundclass);
+    }
+  }
+  if (options & UTF8PROC_CHARBOUND) {
+    utf8proc_bool boundary;
+    int tbc = property->boundclass;
+    boundary = grapheme_break_extended(*last_boundclass, tbc, last_boundclass);
+    if (boundary) {
+      if (bufsize >= 1) dst[0] = 0xFFFF;
+      if (bufsize >= 2) dst[1] = uc;
+      return 2;
+    }
+  }
+  if (bufsize >= 1) *dst = uc;
+  return 1;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
+  utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
+) {
+    return utf8proc_decompose_custom(str, strlen, buffer, bufsize, options, NULL, NULL);
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
+  utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options,
+  utf8proc_custom_func custom_func, void *custom_data
+) {
+  /* strlen will be ignored, if UTF8PROC_NULLTERM is set in options */
+  utf8proc_ssize_t wpos = 0;
+  if ((options & UTF8PROC_COMPOSE) && (options & UTF8PROC_DECOMPOSE))
+    return UTF8PROC_ERROR_INVALIDOPTS;
+  if ((options & UTF8PROC_STRIPMARK) &&
+      !(options & UTF8PROC_COMPOSE) && !(options & UTF8PROC_DECOMPOSE))
+    return UTF8PROC_ERROR_INVALIDOPTS;
+  {
+    utf8proc_int32_t uc;
+    utf8proc_ssize_t rpos = 0;
+    utf8proc_ssize_t decomp_result;
+    int boundclass = UTF8PROC_BOUNDCLASS_START;
+    while (1) {
+      if (options & UTF8PROC_NULLTERM) {
+        rpos += utf8proc_iterate(str + rpos, -1, &uc);
+        /* checking of return value is not necessary,
+           as 'uc' is < 0 in case of error */
+        if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
+        if (rpos < 0) return UTF8PROC_ERROR_OVERFLOW;
+        if (uc == 0) break;
+      } else {
+        if (rpos >= strlen) break;
+        rpos += utf8proc_iterate(str + rpos, strlen - rpos, &uc);
+        if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
+      }
+      if (custom_func != NULL) {
+        uc = custom_func(uc, custom_data);   /* user-specified custom mapping */
+      }
+      decomp_result = utf8proc_decompose_char(
+        uc, buffer + wpos, (bufsize > wpos) ? (bufsize - wpos) : 0, options,
+        &boundclass
+      );
+      if (decomp_result < 0) return decomp_result;
+      wpos += decomp_result;
+      /* prohibiting integer overflows due to too long strings: */
+      if (wpos < 0 ||
+          wpos > (utf8proc_ssize_t)(SSIZE_MAX/sizeof(utf8proc_int32_t)/2))
+        return UTF8PROC_ERROR_OVERFLOW;
+    }
+  }
+  if ((options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && bufsize >= wpos) {
+    utf8proc_ssize_t pos = 0;
+    while (pos < wpos-1) {
+      utf8proc_int32_t uc1, uc2;
+      const utf8proc_property_t *property1, *property2;
+      uc1 = buffer[pos];
+      uc2 = buffer[pos+1];
+      property1 = unsafe_get_property(uc1);
+      property2 = unsafe_get_property(uc2);
+      if (property1->combining_class > property2->combining_class &&
+          property2->combining_class > 0) {
+        buffer[pos] = uc2;
+        buffer[pos+1] = uc1;
+        if (pos > 0) pos--; else pos++;
+      } else {
+        pos++;
+      }
+    }
+  }
+  return wpos;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
+  /* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored */
+  if (options & (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS | UTF8PROC_STRIPCC)) {
+    utf8proc_ssize_t rpos;
+    utf8proc_ssize_t wpos = 0;
+    utf8proc_int32_t uc;
+    for (rpos = 0; rpos < length; rpos++) {
+      uc = buffer[rpos];
+      if (uc == 0x000D && rpos < length-1 && buffer[rpos+1] == 0x000A) rpos++;
+      if (uc == 0x000A || uc == 0x000D || uc == 0x0085 ||
+          ((options & UTF8PROC_STRIPCC) && (uc == 0x000B || uc == 0x000C))) {
+        if (options & UTF8PROC_NLF2LS) {
+          if (options & UTF8PROC_NLF2PS) {
+            buffer[wpos++] = 0x000A;
+          } else {
+            buffer[wpos++] = 0x2028;
+          }
+        } else {
+          if (options & UTF8PROC_NLF2PS) {
+            buffer[wpos++] = 0x2029;
+          } else {
+            buffer[wpos++] = 0x0020;
+          }
+        }
+      } else if ((options & UTF8PROC_STRIPCC) &&
+          (uc < 0x0020 || (uc >= 0x007F && uc < 0x00A0))) {
+        if (uc == 0x0009) buffer[wpos++] = 0x0020;
+      } else {
+        buffer[wpos++] = uc;
+      }
+    }
+    length = wpos;
+  }
+  if (options & UTF8PROC_COMPOSE) {
+    utf8proc_int32_t *starter = NULL;
+    utf8proc_int32_t current_char;
+    const utf8proc_property_t *starter_property = NULL, *current_property;
+    utf8proc_propval_t max_combining_class = -1;
+    utf8proc_ssize_t rpos;
+    utf8proc_ssize_t wpos = 0;
+    utf8proc_int32_t composition;
+    for (rpos = 0; rpos < length; rpos++) {
+      current_char = buffer[rpos];
+      current_property = unsafe_get_property(current_char);
+      if (starter && current_property->combining_class > max_combining_class) {
+        /* combination perhaps possible */
+        utf8proc_int32_t hangul_lindex;
+        utf8proc_int32_t hangul_sindex;
+        hangul_lindex = *starter - UTF8PROC_HANGUL_LBASE;
+        if (hangul_lindex >= 0 && hangul_lindex < UTF8PROC_HANGUL_LCOUNT) {
+          utf8proc_int32_t hangul_vindex;
+          hangul_vindex = current_char - UTF8PROC_HANGUL_VBASE;
+          if (hangul_vindex >= 0 && hangul_vindex < UTF8PROC_HANGUL_VCOUNT) {
+            *starter = UTF8PROC_HANGUL_SBASE +
+              (hangul_lindex * UTF8PROC_HANGUL_VCOUNT + hangul_vindex) *
+              UTF8PROC_HANGUL_TCOUNT;
+            starter_property = NULL;
+            continue;
+          }
+        }
+        hangul_sindex = *starter - UTF8PROC_HANGUL_SBASE;
+        if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT &&
+            (hangul_sindex % UTF8PROC_HANGUL_TCOUNT) == 0) {
+          utf8proc_int32_t hangul_tindex;
+          hangul_tindex = current_char - UTF8PROC_HANGUL_TBASE;
+          if (hangul_tindex >= 0 && hangul_tindex < UTF8PROC_HANGUL_TCOUNT) {
+            *starter += hangul_tindex;
+            starter_property = NULL;
+            continue;
+          }
+        }
+        if (!starter_property) {
+          starter_property = unsafe_get_property(*starter);
+        }
+        if (starter_property->comb_index < 0x8000 &&
+            current_property->comb_index != UINT16_MAX &&
+            current_property->comb_index >= 0x8000) {
+          int sidx = starter_property->comb_index;
+          int idx = (current_property->comb_index & 0x3FFF) - utf8proc_combinations[sidx];
+          if (idx >= 0 && idx <= utf8proc_combinations[sidx + 1] ) {
+            idx += sidx + 2;
+            if (current_property->comb_index & 0x4000) {
+              composition = (utf8proc_combinations[idx] << 16) | utf8proc_combinations[idx+1];
+            } else
+              composition = utf8proc_combinations[idx];
+
+            if (composition > 0 && (!(options & UTF8PROC_STABLE) ||
+                !(unsafe_get_property(composition)->comp_exclusion))) {
+              *starter = composition;
+              starter_property = NULL;
+              continue;
+            }
+          }
+        }
+      }
+      buffer[wpos] = current_char;
+      if (current_property->combining_class) {
+        if (current_property->combining_class > max_combining_class) {
+          max_combining_class = current_property->combining_class;
+        }
+      } else {
+        starter = buffer + wpos;
+        starter_property = NULL;
+        max_combining_class = -1;
+      }
+      wpos++;
+    }
+    length = wpos;
+  }
+  return length;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
+  /* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored
+     ASSERT: 'buffer' has one spare byte of free space at the end! */
+  length = utf8proc_normalize_utf32(buffer, length, options);
+  if (length < 0) return length;
+  {
+    utf8proc_ssize_t rpos, wpos = 0;
+    utf8proc_int32_t uc;
+    if (options & UTF8PROC_CHARBOUND) {
+        for (rpos = 0; rpos < length; rpos++) {
+            uc = buffer[rpos];
+            wpos += unsafe_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
+        }
+    } else {
+        for (rpos = 0; rpos < length; rpos++) {
+            uc = buffer[rpos];
+            wpos += utf8proc_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
+        }
+    }
+    ((utf8proc_uint8_t *)buffer)[wpos] = 0;
+    return wpos;
+  }
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
+) {
+    return utf8proc_map_custom(str, strlen, dstptr, options, NULL, NULL);
+}
+
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options,
+  utf8proc_custom_func custom_func, void *custom_data
+) {
+  utf8proc_int32_t *buffer;
+  utf8proc_ssize_t result;
+  *dstptr = NULL;
+  result = utf8proc_decompose_custom(str, strlen, NULL, 0, options, custom_func, custom_data);
+  if (result < 0) return result;
+  buffer = (utf8proc_int32_t *) malloc(result * sizeof(utf8proc_int32_t) + 1);
+  if (!buffer) return UTF8PROC_ERROR_NOMEM;
+  result = utf8proc_decompose_custom(str, strlen, buffer, result, options, custom_func, custom_data);
+  if (result < 0) {
+    free(buffer);
+    return result;
+  }
+  result = utf8proc_reencode(buffer, result, options);
+  if (result < 0) {
+    free(buffer);
+    return result;
+  }
+  {
+    utf8proc_int32_t *newptr;
+    newptr = (utf8proc_int32_t *) realloc(buffer, (size_t)result+1);
+    if (newptr) buffer = newptr;
+  }
+  *dstptr = (utf8proc_uint8_t *)buffer;
+  return result;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
+  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
+    UTF8PROC_DECOMPOSE);
+  return retval;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
+  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
+    UTF8PROC_COMPOSE);
+  return retval;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
+  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
+    UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
+  return retval;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
+  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
+    UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
+  return retval;
+}
--- a/3rdparty/utf8proc/utf8proc.h
+++ b/3rdparty/utf8proc/utf8proc.h
@ -0,0 +1,707 @@
+/*
+ * Copyright (c) 2015 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
+ * Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+
+/**
+ * @mainpage
+ *
+ * utf8proc is a free/open-source (MIT/expat licensed) C library
+ * providing Unicode normalization, case-folding, and other operations
+ * for strings in the UTF-8 encoding, supporting Unicode version
+ * 8.0.0.  See the utf8proc home page (http://julialang.org/utf8proc/)
+ * for downloads and other information, or the source code on github
+ * (https://github.com/JuliaLang/utf8proc).
+ *
+ * For the utf8proc API documentation, see: @ref utf8proc.h
+ *
+ * The features of utf8proc include:
+ *
+ * - Transformation of strings (@ref utf8proc_map) to:
+ *    - decompose (@ref UTF8PROC_DECOMPOSE) or compose (@ref UTF8PROC_COMPOSE) Unicode combining characters (http://en.wikipedia.org/wiki/Combining_character)
+ *    - canonicalize Unicode compatibility characters (@ref UTF8PROC_COMPAT)
+ *    - strip "ignorable" (@ref UTF8PROC_IGNORE) characters, control characters (@ref UTF8PROC_STRIPCC), or combining characters such as accents (@ref UTF8PROC_STRIPMARK)
+ *    - case-folding (@ref UTF8PROC_CASEFOLD)
+ * - Unicode normalization: @ref utf8proc_NFD, @ref utf8proc_NFC, @ref utf8proc_NFKD, @ref utf8proc_NFKC
+ * - Detecting grapheme boundaries (@ref utf8proc_grapheme_break and @ref UTF8PROC_CHARBOUND)
+ * - Character-width computation: @ref utf8proc_charwidth
+ * - Classification of characters by Unicode category: @ref utf8proc_category and @ref utf8proc_category_string
+ * - Encode (@ref utf8proc_encode_char) and decode (@ref utf8proc_iterate) Unicode codepoints to/from UTF-8.
+ */
+
+/** @file */
+
+#ifndef UTF8PROC_H
+#define UTF8PROC_H
+
+/** @name API version
+ *
+ * The utf8proc API version MAJOR.MINOR.PATCH, following
+ * semantic-versioning rules (http://semver.org) based on API
+ * compatibility.
+ *
+ * This is also returned at runtime by @ref utf8proc_version; however, the
+ * runtime version may append a string like "-dev" to the version number
+ * for prerelease versions.
+ *
+ * @note The shared-library version number in the Makefile
+ *       (and CMakeLists.txt, and MANIFEST) may be different,
+ *       being based on ABI compatibility rather than API compatibility.
+ */
+/** @{ */
+/** The MAJOR version number (increased when backwards API compatibility is broken). */
+#define UTF8PROC_VERSION_MAJOR 2
+/** The MINOR version number (increased when new functionality is added in a backwards-compatible manner). */
+#define UTF8PROC_VERSION_MINOR 1
+/** The PATCH version (increased for fixes that do not change the API). */
+#define UTF8PROC_VERSION_PATCH 0
+/** @} */
+
+#include <stdlib.h>
+#include <sys/types.h>
+#if defined(_MSC_VER) && _MSC_VER < 1800
+// MSVC prior to 2013 lacked stdbool.h and inttypes.h
+typedef signed char utf8proc_int8_t;
+typedef unsigned char utf8proc_uint8_t;
+typedef short utf8proc_int16_t;
+typedef unsigned short utf8proc_uint16_t;
+typedef int utf8proc_int32_t;
+typedef unsigned int utf8proc_uint32_t;
+#  ifdef _WIN64
+typedef __int64 utf8proc_ssize_t;
+typedef unsigned __int64 utf8proc_size_t;
+#  else
+typedef int utf8proc_ssize_t;
+typedef unsigned int utf8proc_size_t;
+#  endif
+#  ifndef __cplusplus
+// emulate C99 bool
+typedef unsigned char utf8proc_bool;
+#    ifndef __bool_true_false_are_defined
+#      define false 0
+#      define true 1
+#      define __bool_true_false_are_defined 1
+#    endif
+#  else
+typedef bool utf8proc_bool;
+#  endif
+#else
+#  include <stddef.h>
+#  include <stdbool.h>
+#  include <inttypes.h>
+typedef int8_t utf8proc_int8_t;
+typedef uint8_t utf8proc_uint8_t;
+typedef int16_t utf8proc_int16_t;
+typedef uint16_t utf8proc_uint16_t;
+typedef int32_t utf8proc_int32_t;
+typedef uint32_t utf8proc_uint32_t;
+typedef size_t utf8proc_size_t;
+typedef ptrdiff_t utf8proc_ssize_t;
+typedef bool utf8proc_bool;
+#endif
+#include <limits.h>
+
+#ifdef _WIN32
+#  ifdef UTF8PROC_EXPORTS
+#    ifndef UTF8PROC_DLLEXPORT
+#    define UTF8PROC_DLLEXPORT __declspec(dllexport)
+#    endif
+#  else
+#    ifndef UTF8PROC_DLLEXPORT
+#    define UTF8PROC_DLLEXPORT __declspec(dllimport)
+#    endif
+#  endif
+#elif __GNUC__ >= 4
+#  ifndef UTF8PROC_DLLEXPORT
+#  define UTF8PROC_DLLEXPORT __attribute__ ((visibility("default")))
+#  endif
+#else
+#  ifndef UTF8PROC_DLLEXPORT
+#  define UTF8PROC_DLLEXPORT
+#  endif
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef SSIZE_MAX
+#define SSIZE_MAX ((size_t)SIZE_MAX/2)
+#endif
+
+#ifndef UINT16_MAX
+#  define UINT16_MAX 65535U
+#endif
+
+/**
+ * Option flags used by several functions in the library.
+ */
+typedef enum {
+  /** The given UTF-8 input is NULL terminated. */
+  UTF8PROC_NULLTERM  = (1<<0),
+  /** Unicode Versioning Stability has to be respected. */
+  UTF8PROC_STABLE    = (1<<1),
+  /** Compatibility decomposition (i.e. formatting information is lost). */
+  UTF8PROC_COMPAT    = (1<<2),
+  /** Return a result with decomposed characters. */
+  UTF8PROC_COMPOSE   = (1<<3),
+  /** Return a result with decomposed characters. */
+  UTF8PROC_DECOMPOSE = (1<<4),
+  /** Strip "default ignorable characters" such as SOFT-HYPHEN or ZERO-WIDTH-SPACE. */
+  UTF8PROC_IGNORE    = (1<<5),
+  /** Return an error, if the input contains unassigned codepoints. */
+  UTF8PROC_REJECTNA  = (1<<6),
+  /**
+   * Indicating that NLF-sequences (LF, CRLF, CR, NEL) are representing a
+   * line break, and should be converted to the codepoint for line
+   * separation (LS).
+   */
+  UTF8PROC_NLF2LS    = (1<<7),
+  /**
+   * Indicating that NLF-sequences are representing a paragraph break, and
+   * should be converted to the codepoint for paragraph separation
+   * (PS).
+   */
+  UTF8PROC_NLF2PS    = (1<<8),
+  /** Indicating that the meaning of NLF-sequences is unknown. */
+  UTF8PROC_NLF2LF    = (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS),
+  /** Strips and/or convers control characters.
+   *
+   * NLF-sequences are transformed into space, except if one of the
+   * NLF2LS/PS/LF options is given. HorizontalTab (HT) and FormFeed (FF)
+   * are treated as a NLF-sequence in this case.  All other control
+   * characters are simply removed.
+   */
+  UTF8PROC_STRIPCC   = (1<<9),
+  /**
+   * Performs unicode case folding, to be able to do a case-insensitive
+   * string comparison.
+   */
+  UTF8PROC_CASEFOLD  = (1<<10),
+  /**
+   * Inserts 0xFF bytes at the beginning of each sequence which is
+   * representing a single grapheme cluster (see UAX#29).
+   */
+  UTF8PROC_CHARBOUND = (1<<11),
+  /** Lumps certain characters together.
+   *
+   * E.g. HYPHEN U+2010 and MINUS U+2212 to ASCII "-". See lump.md for details.
+   *
+   * If NLF2LF is set, this includes a transformation of paragraph and
+   * line separators to ASCII line-feed (LF).
+   */
+  UTF8PROC_LUMP      = (1<<12),
+  /** Strips all character markings.
+   *
+   * This includes non-spacing, spacing and enclosing (i.e. accents).
+   * @note This option works only with @ref UTF8PROC_COMPOSE or
+   *       @ref UTF8PROC_DECOMPOSE
+   */
+  UTF8PROC_STRIPMARK = (1<<13),
+} utf8proc_option_t;
+
+/** @name Error codes
+ * Error codes being returned by almost all functions.
+ */
+/** @{ */
+/** Memory could not be allocated. */
+#define UTF8PROC_ERROR_NOMEM -1
+/** The given string is too long to be processed. */
+#define UTF8PROC_ERROR_OVERFLOW -2
+/** The given string is not a legal UTF-8 string. */
+#define UTF8PROC_ERROR_INVALIDUTF8 -3
+/** The @ref UTF8PROC_REJECTNA flag was set and an unassigned codepoint was found. */
+#define UTF8PROC_ERROR_NOTASSIGNED -4
+/** Invalid options have been used. */
+#define UTF8PROC_ERROR_INVALIDOPTS -5
+/** @} */
+
+/* @name Types */
+
+/** Holds the value of a property. */
+typedef utf8proc_int16_t utf8proc_propval_t;
+
+/** Struct containing information about a codepoint. */
+typedef struct utf8proc_property_struct {
+  /**
+   * Unicode category.
+   * @see utf8proc_category_t.
+   */
+  utf8proc_propval_t category;
+  utf8proc_propval_t combining_class;
+  /**
+   * Bidirectional class.
+   * @see utf8proc_bidi_class_t.
+   */
+  utf8proc_propval_t bidi_class;
+  /**
+   * @anchor Decomposition type.
+   * @see utf8proc_decomp_type_t.
+   */
+  utf8proc_propval_t decomp_type;
+  utf8proc_uint16_t decomp_seqindex;
+  utf8proc_uint16_t casefold_seqindex;
+  utf8proc_uint16_t uppercase_seqindex;
+  utf8proc_uint16_t lowercase_seqindex;
+  utf8proc_uint16_t titlecase_seqindex;
+  utf8proc_uint16_t comb_index;
+  unsigned bidi_mirrored:1;
+  unsigned comp_exclusion:1;
+  /**
+   * Can this codepoint be ignored?
+   *
+   * Used by @ref utf8proc_decompose_char when @ref UTF8PROC_IGNORE is
+   * passed as an option.
+   */
+  unsigned ignorable:1;
+  unsigned control_boundary:1;
+  /** The width of the codepoint. */
+  unsigned charwidth:2;
+  unsigned pad:2;
+  /**
+   * Boundclass.
+   * @see utf8proc_boundclass_t.
+   */
+  unsigned boundclass:8;
+} utf8proc_property_t;
+
+/** Unicode categories. */
+typedef enum {
+  UTF8PROC_CATEGORY_CN  = 0, /**< Other, not assigned */
+  UTF8PROC_CATEGORY_LU  = 1, /**< Letter, uppercase */
+  UTF8PROC_CATEGORY_LL  = 2, /**< Letter, lowercase */
+  UTF8PROC_CATEGORY_LT  = 3, /**< Letter, titlecase */
+  UTF8PROC_CATEGORY_LM  = 4, /**< Letter, modifier */
+  UTF8PROC_CATEGORY_LO  = 5, /**< Letter, other */
+  UTF8PROC_CATEGORY_MN  = 6, /**< Mark, nonspacing */
+  UTF8PROC_CATEGORY_MC  = 7, /**< Mark, spacing combining */
+  UTF8PROC_CATEGORY_ME  = 8, /**< Mark, enclosing */
+  UTF8PROC_CATEGORY_ND  = 9, /**< Number, decimal digit */
+  UTF8PROC_CATEGORY_NL = 10, /**< Number, letter */
+  UTF8PROC_CATEGORY_NO = 11, /**< Number, other */
+  UTF8PROC_CATEGORY_PC = 12, /**< Punctuation, connector */
+  UTF8PROC_CATEGORY_PD = 13, /**< Punctuation, dash */
+  UTF8PROC_CATEGORY_PS = 14, /**< Punctuation, open */
+  UTF8PROC_CATEGORY_PE = 15, /**< Punctuation, close */
+  UTF8PROC_CATEGORY_PI = 16, /**< Punctuation, initial quote */
+  UTF8PROC_CATEGORY_PF = 17, /**< Punctuation, final quote */
+  UTF8PROC_CATEGORY_PO = 18, /**< Punctuation, other */
+  UTF8PROC_CATEGORY_SM = 19, /**< Symbol, math */
+  UTF8PROC_CATEGORY_SC = 20, /**< Symbol, currency */
+  UTF8PROC_CATEGORY_SK = 21, /**< Symbol, modifier */
+  UTF8PROC_CATEGORY_SO = 22, /**< Symbol, other */
+  UTF8PROC_CATEGORY_ZS = 23, /**< Separator, space */
+  UTF8PROC_CATEGORY_ZL = 24, /**< Separator, line */
+  UTF8PROC_CATEGORY_ZP = 25, /**< Separator, paragraph */
+  UTF8PROC_CATEGORY_CC = 26, /**< Other, control */
+  UTF8PROC_CATEGORY_CF = 27, /**< Other, format */
+  UTF8PROC_CATEGORY_CS = 28, /**< Other, surrogate */
+  UTF8PROC_CATEGORY_CO = 29, /**< Other, private use */
+} utf8proc_category_t;
+
+/** Bidirectional character classes. */
+typedef enum {
+  UTF8PROC_BIDI_CLASS_L     = 1, /**< Left-to-Right */
+  UTF8PROC_BIDI_CLASS_LRE   = 2, /**< Left-to-Right Embedding */
+  UTF8PROC_BIDI_CLASS_LRO   = 3, /**< Left-to-Right Override */
+  UTF8PROC_BIDI_CLASS_R     = 4, /**< Right-to-Left */
+  UTF8PROC_BIDI_CLASS_AL    = 5, /**< Right-to-Left Arabic */
+  UTF8PROC_BIDI_CLASS_RLE   = 6, /**< Right-to-Left Embedding */
+  UTF8PROC_BIDI_CLASS_RLO   = 7, /**< Right-to-Left Override */
+  UTF8PROC_BIDI_CLASS_PDF   = 8, /**< Pop Directional Format */
+  UTF8PROC_BIDI_CLASS_EN    = 9, /**< European Number */
+  UTF8PROC_BIDI_CLASS_ES   = 10, /**< European Separator */
+  UTF8PROC_BIDI_CLASS_ET   = 11, /**< European Number Terminator */
+  UTF8PROC_BIDI_CLASS_AN   = 12, /**< Arabic Number */
+  UTF8PROC_BIDI_CLASS_CS   = 13, /**< Common Number Separator */
+  UTF8PROC_BIDI_CLASS_NSM  = 14, /**< Nonspacing Mark */
+  UTF8PROC_BIDI_CLASS_BN   = 15, /**< Boundary Neutral */
+  UTF8PROC_BIDI_CLASS_B    = 16, /**< Paragraph Separator */
+  UTF8PROC_BIDI_CLASS_S    = 17, /**< Segment Separator */
+  UTF8PROC_BIDI_CLASS_WS   = 18, /**< Whitespace */
+  UTF8PROC_BIDI_CLASS_ON   = 19, /**< Other Neutrals */
+  UTF8PROC_BIDI_CLASS_LRI  = 20, /**< Left-to-Right Isolate */
+  UTF8PROC_BIDI_CLASS_RLI  = 21, /**< Right-to-Left Isolate */
+  UTF8PROC_BIDI_CLASS_FSI  = 22, /**< First Strong Isolate */
+  UTF8PROC_BIDI_CLASS_PDI  = 23, /**< Pop Directional Isolate */
+} utf8proc_bidi_class_t;
+
+/** Decomposition type. */
+typedef enum {
+  UTF8PROC_DECOMP_TYPE_FONT      = 1, /**< Font */
+  UTF8PROC_DECOMP_TYPE_NOBREAK   = 2, /**< Nobreak */
+  UTF8PROC_DECOMP_TYPE_INITIAL   = 3, /**< Initial */
+  UTF8PROC_DECOMP_TYPE_MEDIAL    = 4, /**< Medial */
+  UTF8PROC_DECOMP_TYPE_FINAL     = 5, /**< Final */
+  UTF8PROC_DECOMP_TYPE_ISOLATED  = 6, /**< Isolated */
+  UTF8PROC_DECOMP_TYPE_CIRCLE    = 7, /**< Circle */
+  UTF8PROC_DECOMP_TYPE_SUPER     = 8, /**< Super */
+  UTF8PROC_DECOMP_TYPE_SUB       = 9, /**< Sub */
+  UTF8PROC_DECOMP_TYPE_VERTICAL = 10, /**< Vertical */
+  UTF8PROC_DECOMP_TYPE_WIDE     = 11, /**< Wide */
+  UTF8PROC_DECOMP_TYPE_NARROW   = 12, /**< Narrow */
+  UTF8PROC_DECOMP_TYPE_SMALL    = 13, /**< Small */
+  UTF8PROC_DECOMP_TYPE_SQUARE   = 14, /**< Square */
+  UTF8PROC_DECOMP_TYPE_FRACTION = 15, /**< Fraction */
+  UTF8PROC_DECOMP_TYPE_COMPAT   = 16, /**< Compat */
+} utf8proc_decomp_type_t;
+
+/** Boundclass property. (TR29) */
+typedef enum {
+  UTF8PROC_BOUNDCLASS_START              =  0, /**< Start */
+  UTF8PROC_BOUNDCLASS_OTHER              =  1, /**< Other */
+  UTF8PROC_BOUNDCLASS_CR                 =  2, /**< Cr */
+  UTF8PROC_BOUNDCLASS_LF                 =  3, /**< Lf */
+  UTF8PROC_BOUNDCLASS_CONTROL            =  4, /**< Control */
+  UTF8PROC_BOUNDCLASS_EXTEND             =  5, /**< Extend */
+  UTF8PROC_BOUNDCLASS_L                  =  6, /**< L */
+  UTF8PROC_BOUNDCLASS_V                  =  7, /**< V */
+  UTF8PROC_BOUNDCLASS_T                  =  8, /**< T */
+  UTF8PROC_BOUNDCLASS_LV                 =  9, /**< Lv */
+  UTF8PROC_BOUNDCLASS_LVT                = 10, /**< Lvt */
+  UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR = 11, /**< Regional indicator */
+  UTF8PROC_BOUNDCLASS_SPACINGMARK        = 12, /**< Spacingmark */
+  UTF8PROC_BOUNDCLASS_PREPEND            = 13, /**< Prepend */
+  UTF8PROC_BOUNDCLASS_ZWJ                = 14, /**< Zero Width Joiner */
+  UTF8PROC_BOUNDCLASS_E_BASE             = 15, /**< Emoji Base */
+  UTF8PROC_BOUNDCLASS_E_MODIFIER         = 16, /**< Emoji Modifier */
+  UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ     = 17, /**< Glue_After_ZWJ */
+  UTF8PROC_BOUNDCLASS_E_BASE_GAZ         = 18, /**< E_BASE + GLUE_AFTER_ZJW */
+} utf8proc_boundclass_t;
+
+/**
+ * Function pointer type passed to @ref utf8proc_map_custom and
+ * @ref utf8proc_decompose_custom, which is used to specify a user-defined
+ * mapping of codepoints to be applied in conjunction with other mappings.
+ */
+typedef utf8proc_int32_t (*utf8proc_custom_func)(utf8proc_int32_t codepoint, void *data);
+
+/**
+ * Array containing the byte lengths of a UTF-8 encoded codepoint based
+ * on the first byte.
+ */
+UTF8PROC_DLLEXPORT extern const utf8proc_int8_t utf8proc_utf8class[256];
+
+/**
+ * Returns the utf8proc API version as a string MAJOR.MINOR.PATCH
+ * (http://semver.org format), possibly with a "-dev" suffix for
+ * development versions.
+ */
+UTF8PROC_DLLEXPORT const char *utf8proc_version(void);
+
+/**
+ * Returns an informative error string for the given utf8proc error code
+ * (e.g. the error codes returned by @ref utf8proc_map).
+ */
+UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode);
+
+/**
+ * Reads a single codepoint from the UTF-8 sequence being pointed to by `str`.
+ * The maximum number of bytes read is `strlen`, unless `strlen` is
+ * negative (in which case up to 4 bytes are read).
+ *
+ * If a valid codepoint could be read, it is stored in the variable
+ * pointed to by `codepoint_ref`, otherwise that variable will be set to -1.
+ * In case of success, the number of bytes read is returned; otherwise, a
+ * negative error code is returned.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *codepoint_ref);
+
+/**
+ * Check if a codepoint is valid (regardless of whether it has been
+ * assigned a value by the current Unicode standard).
+ *
+ * @return 1 if the given `codepoint` is valid and otherwise return 0.
+ */
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t codepoint);
+
+/**
+ * Encodes the codepoint as an UTF-8 string in the byte array pointed
+ * to by `dst`. This array must be at least 4 bytes long.
+ *
+ * In case of success the number of bytes written is returned, and
+ * otherwise 0 is returned.
+ *
+ * This function does not check whether `codepoint` is valid Unicode.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t codepoint, utf8proc_uint8_t *dst);
+
+/**
+ * Look up the properties for a given codepoint.
+ *
+ * @param codepoint The Unicode codepoint.
+ *
+ * @returns
+ * A pointer to a (constant) struct containing information about
+ * the codepoint.
+ * @par
+ * If the codepoint is unassigned or invalid, a pointer to a special struct is
+ * returned in which `category` is 0 (@ref UTF8PROC_CATEGORY_CN).
+ */
+UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t codepoint);
+
+/** Decompose a codepoint into an array of codepoints.
+ *
+ * @param codepoint the codepoint.
+ * @param dst the destination buffer.
+ * @param bufsize the size of the destination buffer.
+ * @param options one or more of the following flags:
+ * - @ref UTF8PROC_REJECTNA  - return an error `codepoint` is unassigned
+ * - @ref UTF8PROC_IGNORE    - strip "default ignorable" codepoints
+ * - @ref UTF8PROC_CASEFOLD  - apply Unicode casefolding
+ * - @ref UTF8PROC_COMPAT    - replace certain codepoints with their
+ *                             compatibility decomposition
+ * - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
+ * - @ref UTF8PROC_LUMP      - lump certain different codepoints together
+ * - @ref UTF8PROC_STRIPMARK - remove all character marks
+ * @param last_boundclass
+ * Pointer to an integer variable containing
+ * the previous codepoint's boundary class if the @ref UTF8PROC_CHARBOUND
+ * option is used.  Otherwise, this parameter is ignored.
+ *
+ * @return
+ * In case of success, the number of codepoints written is returned; in case
+ * of an error, a negative error code is returned (@ref utf8proc_errmsg).
+ * @par
+ * If the number of written codepoints would be bigger than `bufsize`, the
+ * required buffer size is returned, while the buffer will be overwritten with
+ * undefined data.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(
+  utf8proc_int32_t codepoint, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize,
+  utf8proc_option_t options, int *last_boundclass
+);
+
+/**
+ * The same as @ref utf8proc_decompose_char, but acts on a whole UTF-8
+ * string and orders the decomposed sequences correctly.
+ *
+ * If the @ref UTF8PROC_NULLTERM flag in `options` is set, processing
+ * will be stopped, when a NULL byte is encounted, otherwise `strlen`
+ * bytes are processed.  The result (in the form of 32-bit unicode
+ * codepoints) is written into the buffer being pointed to by
+ * `buffer` (which must contain at least `bufsize` entries).  In case of
+ * success, the number of codepoints written is returned; in case of an
+ * error, a negative error code is returned (@ref utf8proc_errmsg).
+ * See @ref utf8proc_decompose_custom to supply additional transformations.
+ *
+ * If the number of written codepoints would be bigger than `bufsize`, the
+ * required buffer size is returned, while the buffer will be overwritten with
+ * undefined data.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
+  utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
+);
+
+/**
+ * The same as @ref utf8proc_decompose, but also takes a `custom_func` mapping function
+ * that is called on each codepoint in `str` before any other transformations
+ * (along with a `custom_data` pointer that is passed through to `custom_func`).
+ * The `custom_func` argument is ignored if it is `NULL`.  See also @ref utf8proc_map_custom.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
+  utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options,
+  utf8proc_custom_func custom_func, void *custom_data
+);
+
+/**
+ * Normalizes the sequence of `length` codepoints pointed to by `buffer`
+ * in-place (i.e., the result is also stored in `buffer`).
+ *
+ * @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
+ * @param length the length (in codepoints) of the buffer.
+ * @param options a bitwise or (`|`) of one or more of the following flags:
+ * - @ref UTF8PROC_NLF2LS  - convert LF, CRLF, CR and NEL into LS
+ * - @ref UTF8PROC_NLF2PS  - convert LF, CRLF, CR and NEL into PS
+ * - @ref UTF8PROC_NLF2LF  - convert LF, CRLF, CR and NEL into LF
+ * - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
+ * - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
+ *                           codepoints
+ * - @ref UTF8PROC_STABLE  - prohibit combining characters that would violate
+ *                           the unicode versioning stability
+ *
+ * @return
+ * In case of success, the length (in codepoints) of the normalized UTF-32 string is
+ * returned; otherwise, a negative error code is returned (@ref utf8proc_errmsg).
+ *
+ * @warning The entries of the array pointed to by `str` have to be in the
+ *          range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
+
+/**
+ * Reencodes the sequence of `length` codepoints pointed to by `buffer`
+ * UTF-8 data in-place (i.e., the result is also stored in `buffer`).
+ * Can optionally normalize the UTF-32 sequence prior to UTF-8 conversion.
+ *
+ * @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
+ * @param length the length (in codepoints) of the buffer.
+ * @param options a bitwise or (`|`) of one or more of the following flags:
+ * - @ref UTF8PROC_NLF2LS  - convert LF, CRLF, CR and NEL into LS
+ * - @ref UTF8PROC_NLF2PS  - convert LF, CRLF, CR and NEL into PS
+ * - @ref UTF8PROC_NLF2LF  - convert LF, CRLF, CR and NEL into LF
+ * - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
+ * - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
+ *                           codepoints
+ * - @ref UTF8PROC_STABLE  - prohibit combining characters that would violate
+ *                           the unicode versioning stability
+ * - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
+ *
+ * @return
+ * In case of success, the length (in bytes) of the resulting nul-terminated
+ * UTF-8 string is returned; otherwise, a negative error code is returned
+ * (@ref utf8proc_errmsg).
+ *
+ * @warning The amount of free space pointed to by `buffer` must
+ *          exceed the amount of the input data by one byte, and the
+ *          entries of the array pointed to by `str` have to be in the
+ *          range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
+
+/**
+ * Given a pair of consecutive codepoints, return whether a grapheme break is
+ * permitted between them (as defined by the extended grapheme clusters in UAX#29).
+ *
+ * @param state Beginning with Version 29 (Unicode 9.0.0), this algorithm requires
+ *              state to break graphemes. This state can be passed in as a pointer
+ *              in the `state` argument and should initially be set to 0. If the
+ *              state is not passed in (i.e. a null pointer is passed), UAX#29 rules
+ *              GB10/12/13 which require this state will not be applied, essentially
+ *              matching the rules in Unicode 8.0.0.
+ *
+ * @warning If the state parameter is used, `utf8proc_grapheme_break_stateful` must
+ *          be called IN ORDER on ALL potential breaks in a string.
+ */
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
+    utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2, utf8proc_int32_t *state);
+
+/**
+ * Same as @ref utf8proc_grapheme_break_stateful, except without support for the
+ * Unicode 9 additions to the algorithm. Supported for legacy reasons.
+ */
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
+    utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2);
+
+
+/**
+ * Given a codepoint `c`, return the codepoint of the corresponding
+ * lower-case character, if any; otherwise (if there is no lower-case
+ * variant, or if `c` is not a valid codepoint) return `c`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c);
+
+/**
+ * Given a codepoint `c`, return the codepoint of the corresponding
+ * upper-case character, if any; otherwise (if there is no upper-case
+ * variant, or if `c` is not a valid codepoint) return `c`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c);
+
+/**
+ * Given a codepoint `c`, return the codepoint of the corresponding
+ * title-case character, if any; otherwise (if there is no title-case
+ * variant, or if `c` is not a valid codepoint) return `c`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c);
+
+/**
+ * Given a codepoint, return a character width analogous to `wcwidth(codepoint)`,
+ * except that a width of 0 is returned for non-printable codepoints
+ * instead of -1 as in `wcwidth`.
+ *
+ * @note
+ * If you want to check for particular types of non-printable characters,
+ * (analogous to `isprint` or `iscntrl`), use @ref utf8proc_category. */
+UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t codepoint);
+
+/**
+ * Return the Unicode category for the codepoint (one of the
+ * @ref utf8proc_category_t constants.)
+ */
+UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t codepoint);
+
+/**
+ * Return the two-letter (nul-terminated) Unicode category string for
+ * the codepoint (e.g. `"Lu"` or `"Co"`).
+ */
+UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t codepoint);
+
+/**
+ * Maps the given UTF-8 string pointed to by `str` to a new UTF-8
+ * string, allocated dynamically by `malloc` and returned via `dstptr`.
+ *
+ * If the @ref UTF8PROC_NULLTERM flag in the `options` field is set,
+ * the length is determined by a NULL terminator, otherwise the
+ * parameter `strlen` is evaluated to determine the string length, but
+ * in any case the result will be NULL terminated (though it might
+ * contain NULL characters with the string if `str` contained NULL
+ * characters). Other flags in the `options` field are passed to the
+ * functions defined above, and regarded as described.  See also
+ * @ref utfproc_map_custom to supply a custom codepoint transformation.
+ *
+ * In case of success the length of the new string is returned,
+ * otherwise a negative error code is returned.
+ *
+ * @note The memory of the new UTF-8 string will have been allocated
+ * with `malloc`, and should therefore be deallocated with `free`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
+);
+
+/**
+ * Like @ref utf8proc_map, but also takes a `custom_func` mapping function
+ * that is called on each codepoint in `str` before any other transformations
+ * (along with a `custom_data` pointer that is passed through to `custom_func`).
+ * The `custom_func` argument is ignored if it is `NULL`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options,
+  utf8proc_custom_func custom_func, void *custom_data
+);
+
+/** @name Unicode normalization
+ *
+ * Returns a pointer to newly allocated memory of a NFD, NFC, NFKD or NFKC
+ * normalized version of the null-terminated string `str`.  These
+ * are shortcuts to calling @ref utf8proc_map with @ref UTF8PROC_NULLTERM
+ * combined with @ref UTF8PROC_STABLE and flags indicating the normalization.
+ */
+/** @{ */
+/** NFD normalization (@ref UTF8PROC_DECOMPOSE). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str);
+/** NFC normalization (@ref UTF8PROC_COMPOSE). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str);
+/** NFKD normalization (@ref UTF8PROC_DECOMPOSE and @ref UTF8PROC_COMPAT). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str);
+/** NFKC normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str);
+/** @} */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
--- a/3rdparty/utf8proc/utf8proc_data.c
+++ b/3rdparty/utf8proc/utf8proc_data.c
--- a/3rdparty/utf8proc/utils.cmake
+++ b/3rdparty/utf8proc/utils.cmake
@ -0,0 +1,20 @@
+
+function (disallow_intree_builds)
+  # Adapted from LLVM's toplevel CMakeLists.txt file
+  if( CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR AND NOT MSVC_IDE )
+    message(FATAL_ERROR "
+      In-source builds are not allowed. CMake would overwrite the
+      makefiles distributed with utf8proc. Please create a directory
+      and run cmake from there. Building in a subdirectory is
+      fine, e.g.:
+      
+        mkdir build
+        cd build
+        cmake ..
+      
+      This process created the file `CMakeCache.txt' and the
+      directory `CMakeFiles'. Please delete them.
+      
+      ")
+  endif()
+endfunction()
--- a/scripts/src/3rdparty.lua
+++ b/scripts/src/3rdparty.lua
@ -1585,3 +1585,42 @@ project "linenoise-ng"
 		MAME_DIR .. "3rdparty/linenoise-ng/src/wcwidth.cpp",
 	}
 end
+
+
+--------------------------------------------------
+-- utf8proc library objects
+--------------------------------------------------
+
+if not _OPTIONS["with-system-utf8proc"] then
+project "utf8proc"
+	uuid "1f881f09-0395-4483-ac37-2935fb092187"
+	kind "StaticLib"
+
+  defines {
+    "UTF8PROC_DLLEXPORT="
+  }
+  
+	configuration "Debug"
+		defines {
+			"verbose=-1",
+		}
+
+	configuration { "gmake or ninja" }
+		buildoptions_c {
+			"-Wno-strict-prototypes",
+		}
+
+	configuration { }
+		defines {
+			"ZLIB_CONST",
+		}
+
+	files {
+		MAME_DIR .. "3rdparty/utf8proc/utf8proc.c"
+	}
+else
+links {
+	ext_lib("utf8proc"),
+}
+end
+
--- a/scripts/src/main.lua
+++ b/scripts/src/main.lua
@ -276,6 +276,7 @@ end
 	links {
 		ext_lib("zlib"),
 		ext_lib("flac"),
+		ext_lib("utf8proc"),
 	}
 if (STANDALONE~=true) then
 	links {
--- a/scripts/src/tools.lua
+++ b/scripts/src/tools.lua
@ -31,6 +31,7 @@ links {
 	"7z",
 	"ocore_" .. _OPTIONS["osd"],
 	ext_lib("zlib"),
+	ext_lib("utf8proc"),
 }

 includedirs {
@ -73,6 +74,7 @@ links {
 	"ocore_" .. _OPTIONS["osd"],
 	ext_lib("zlib"),
 	ext_lib("flac"),
+	ext_lib("utf8proc"),
 }

 includedirs {
@ -118,6 +120,7 @@ links {
 	ext_lib("expat"),
 	"ocore_" .. _OPTIONS["osd"],
 	ext_lib("zlib"),
+	ext_lib("utf8proc"),
 }

 includedirs {
@ -161,6 +164,7 @@ links {
 	"ocore_" .. _OPTIONS["osd"],
 	ext_lib("zlib"),
 	ext_lib("flac"),
+	ext_lib("utf8proc"),
 }

 includedirs {
@ -206,6 +210,7 @@ links {
 	"ocore_" .. _OPTIONS["osd"],
 	ext_lib("zlib"),
 	ext_lib("flac"),
+	ext_lib("utf8proc"),
 }

 includedirs {
@ -252,6 +257,7 @@ links {
 	"ocore_" .. _OPTIONS["osd"],
 	ext_lib("zlib"),
 	ext_lib("flac"),
+	ext_lib("utf8proc"),
 }

 includedirs {
@ -296,6 +302,7 @@ links {
 	ext_lib("expat"),
 	"ocore_" .. _OPTIONS["osd"],
 	ext_lib("zlib"),
+	ext_lib("utf8proc"),
 }

 includedirs {
@ -336,6 +343,7 @@ links {
 	ext_lib("expat"),
 	"ocore_" .. _OPTIONS["osd"],
 	ext_lib("zlib"),
+	ext_lib("utf8proc"),
 }

 includedirs {
@ -376,6 +384,7 @@ links {
 	ext_lib("expat"),
 	"ocore_" .. _OPTIONS["osd"],
 	ext_lib("zlib"),
+	ext_lib("utf8proc"),
 }

 includedirs {
@ -418,6 +427,7 @@ links {
 	"ocore_" .. _OPTIONS["osd"],
 	ext_lib("zlib"),
 	ext_lib("flac"),
+	ext_lib("utf8proc"),
 }

 includedirs {
@ -458,6 +468,7 @@ links {
 	ext_lib("expat"),
 	"ocore_" .. _OPTIONS["osd"],
 	ext_lib("zlib"),
+	ext_lib("utf8proc"),
 }

 includedirs {
@ -501,6 +512,7 @@ links {
 	"netlist",
 	ext_lib("zlib"),
 	ext_lib("flac"),
+	ext_lib("utf8proc"),
 }

 includedirs {
@ -585,6 +597,7 @@ links {
 	"ocore_" .. _OPTIONS["osd"],
 	ext_lib("zlib"),
 	ext_lib("flac"),
+	ext_lib("utf8proc"),
 }

 includedirs {
@ -630,6 +643,7 @@ links {
 	"ocore_" .. _OPTIONS["osd"],
 	ext_lib("zlib"),
 	ext_lib("flac"),
+	ext_lib("utf8proc"),
 }

 includedirs {
@ -675,6 +689,7 @@ links {
 	"ocore_" .. _OPTIONS["osd"],
 	ext_lib("zlib"),
 	ext_lib("flac"),
+	ext_lib("utf8proc"),
 }

 includedirs {
--- a/src/lib/util/unicode.cpp
+++ b/src/lib/util/unicode.cpp
@ -2,7 +2,7 @@
 // copyright-holders:Aaron Giles
 /*********************************************************************

-    unicode.c
+    unicode.cpp

    Unicode related functions

@ -10,6 +10,12 @@

 #include "unicode.h"

+#ifdef _WIN32
+#define UTF8PROC_DLLEXPORT
+#endif
+
+#include "utf8proc/utf8proc.h"
+

 //-------------------------------------------------
 //  uchar_isvalid - return true if a given
@ -345,6 +351,87 @@ int utf16f_from_uchar(char16_t *utf16string, size_t count, char32_t uchar)
 }


+//-------------------------------------------------
+//  internal_normalize_unicode - uses utf8proc to
+//	normalize unicode
+//-------------------------------------------------
+
+static std::string internal_normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form, bool null_terminated)
+{
+	// convert the normalization form
+	int options;
+	switch (normalization_form)
+	{
+	case unicode_normalization_form::C:
+		options = UTF8PROC_STABLE | UTF8PROC_COMPOSE;
+		break;
+	case unicode_normalization_form::D:
+		options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE;
+		break;
+	case unicode_normalization_form::KC:
+		options = UTF8PROC_STABLE | UTF8PROC_COMPOSE | UTF8PROC_COMPAT;
+		break;
+	case unicode_normalization_form::KD:
+		options = UTF8PROC_STABLE | UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT;
+		break;
+	default:
+		throw false;
+	}
+
+	// was this null terminated?
+	if (null_terminated)
+		options |= UTF8PROC_NULLTERM;
+
+	// invoke utf8proc
+	utf8proc_uint8_t *utf8proc_result;
+	utf8proc_ssize_t utf8proc_result_length = utf8proc_map((utf8proc_uint8_t *) s, length, &utf8proc_result, (utf8proc_option_t)options);
+
+	// conver the result
+	std::string result;
+	if (utf8proc_result)
+	{
+		if (utf8proc_result_length > 0)
+			result = std::string((const char *)utf8proc_result, utf8proc_result_length);
+		free(utf8proc_result);
+	}
+
+	return result;
+}
+
+
+//-------------------------------------------------
+//  normalize_unicode - uses utf8proc to normalize
+//	unicode
+//-------------------------------------------------
+
+std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form)
+{
+	return internal_normalize_unicode(s.c_str(), s.length(), normalization_form, false);
+}
+
+
+//-------------------------------------------------
+//  normalize_unicode - uses utf8proc to normalize
+//	unicode
+//-------------------------------------------------
+
+std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form)
+{
+	return internal_normalize_unicode(s, 0, normalization_form, true);
+}
+
+
+//-------------------------------------------------
+//  normalize_unicode - uses utf8proc to normalize
+//	unicode
+//-------------------------------------------------
+
+std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form)
+{
+	return internal_normalize_unicode(s, length, normalization_form, false);
+}
+
+
 //-------------------------------------------------
 //  utf8_previous_char - return a pointer to the
 //  previous character in a string
--- a/src/lib/util/unicode.h
+++ b/src/lib/util/unicode.h
@ -72,6 +72,9 @@
 #define UTF8_UP                 "\xe2\x86\x91"      /* cursor up */
 #define UTF8_DOWN               "\xe2\x86\x93"      /* cursor down */

+enum class unicode_normalization_form { C, D, KC, KD };
+
+

 /***************************************************************************
    FUNCTION PROTOTYPES
@ -97,6 +100,11 @@ std::string utf8_from_uchar(char32_t uchar);
 int utf16_from_uchar(char16_t *utf16string, size_t count, char32_t uchar);
 int utf16f_from_uchar(char16_t *utf16string, size_t count, char32_t uchar);

+// unicode normalization
+std::string normalize_unicode(const std::string &s, unicode_normalization_form normalization_form);
+std::string normalize_unicode(const char *s, unicode_normalization_form normalization_form);
+std::string normalize_unicode(const char *s, size_t length, unicode_normalization_form normalization_form);
+
 // misc UTF-8 helpers
 const char *utf8_previous_char(const char *utf8string);
 bool utf8_is_valid_string(const char *utf8string);