From 14f4e0e2718c32147a1f99416f3bec0221ce3dad Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Sun, 16 Mar 2014 16:49:31 -0700 Subject: [PATCH 01/13] Fix for issue where pager contents may stay around if you executed a command with pager contents visible --- reader.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/reader.cpp b/reader.cpp index 069771c17..7bbac43a9 100644 --- a/reader.cpp +++ b/reader.cpp @@ -3427,6 +3427,9 @@ const wchar_t *reader_readline(void) break; } + /* The user may have hit return with pager contents, but while not navigating them. Clear the pager in that event. */ + clear_pager(); + /* We only execute the command line */ editable_line_t *el = &data->command_line; From 5f118542868c7e627a1f45c925076d35e2e6c18b Mon Sep 17 00:00:00 2001 From: jer-gentoo Date: Mon, 3 Mar 2014 01:58:07 +0100 Subject: [PATCH 02/13] Check for libtinfo after libncurses See Gentoo bug 459768 (https://bugs.gentoo.org/show_bug.cgi?id=459768) Closes #1322 (https://github.com/fish-shell/fish-shell/pull/1322). --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 342495668..b5ba9f0ba 100644 --- a/configure.ac +++ b/configure.ac @@ -407,7 +407,7 @@ AC_DEFINE( AC_SEARCH_LIBS( connect, socket, , [AC_MSG_ERROR([Cannot find the socket library, needed to build this package.] )] ) AC_SEARCH_LIBS( nanosleep, rt, , [AC_MSG_ERROR([Cannot find the rt library, needed to build this package.] )] ) AC_SEARCH_LIBS( pthread_create, pthread, , [AC_MSG_ERROR([Cannot find the pthread library, needed to build this package.] )] ) -AC_SEARCH_LIBS( setupterm, [ncurses curses], , [AC_MSG_ERROR([Could not find a curses implementation, needed to build fish. If this is Linux, try running 'sudo apt-get install libncurses5-dev' or 'sudo yum install ncurses-devel'])] ) +AC_SEARCH_LIBS( setupterm, [ncurses tinfo curses], , [AC_MSG_ERROR([Could not find a curses implementation, needed to build fish. If this is Linux, try running 'sudo apt-get install libncurses5-dev' or 'sudo yum install ncurses-devel'])] ) AC_SEARCH_LIBS( [nan], [m], [AC_DEFINE( [HAVE_NAN], [1], [Define to 1 if you have the nan function])] ) if test x$local_gettext != xno; then From 06eb271bda3a0ef832d9a1aa1d3e84b2d39e5dfa Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Sat, 22 Mar 2014 14:46:23 -0700 Subject: [PATCH 03/13] Changes full_escape to not generate \x escapes for non-ASCII characters. Partially reverts 51de269 . Fixes #1225 --- env_universal_common.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/env_universal_common.cpp b/env_universal_common.cpp index dbf79c1a6..c175b4a14 100644 --- a/env_universal_common.cpp +++ b/env_universal_common.cpp @@ -756,8 +756,9 @@ static wcstring full_escape(const wchar_t *in) { out.push_back(c); } - else if (c < 256) + else if (c <= ASCII_MAX) { + // See #1225 for discussion of use of ASCII_MAX here append_format(out, L"\\x%.2x", c); } else if (c < 65536) From aabed8279e4a086cf953006023bc14ec1d1d83b8 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Sat, 22 Mar 2014 23:46:58 -0700 Subject: [PATCH 04/13] Incorporate a modified UTF8 <-> wchar_t implementation from Alexey Vatchenko (http://www.bsdua.org/libbsdua.html) in preparation for eliminating our dependency on iconv --- Makefile.in | 4 +- doc_src/license.hdr | 16 + fish.xcodeproj/project.pbxproj | 8 + fish_tests.cpp | 307 ++++++++++++++++++-- utf8.cpp | 514 +++++++++++++++++++++++++++++++++ utf8.h | 39 +++ 6 files changed, 861 insertions(+), 27 deletions(-) create mode 100644 utf8.cpp create mode 100644 utf8.h diff --git a/Makefile.in b/Makefile.in index 8d18ce1d7..3063438c2 100644 --- a/Makefile.in +++ b/Makefile.in @@ -91,7 +91,7 @@ FISH_OBJS := function.o builtin.o complete.o env.o exec.o expand.o \ signal.o io.o parse_util.o common.o screen.o path.o autoload.o \ parser_keywords.o iothread.o color.o postfork.o \ builtin_test.o parse_tree.o parse_productions.o parse_execution.cpp \ - pager.cpp + pager.cpp utf8.o FISH_INDENT_OBJS := fish_indent.o print_help.o common.o \ parser_keywords.o wutil.o tokenizer.o @@ -117,7 +117,7 @@ FISH_TESTS_OBJS := $(FISH_OBJS) fish_tests.o # FISHD_OBJS := fishd.o env_universal_common.o wutil.o print_help.o \ - common.o + common.o utf8.o # diff --git a/doc_src/license.hdr b/doc_src/license.hdr index c07a94ad2..7f14383a7 100644 --- a/doc_src/license.hdr +++ b/doc_src/license.hdr @@ -1402,4 +1402,20 @@ POSSIBILITY OF SUCH DAMAGES. */ +

License for UTF8

+ +

Copyright (c) 2007 Alexey Vatchenko + +

Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +

THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + \htmlonly \endhtmlonly diff --git a/fish.xcodeproj/project.pbxproj b/fish.xcodeproj/project.pbxproj index d51f53e3b..47d04c4cc 100644 --- a/fish.xcodeproj/project.pbxproj +++ b/fish.xcodeproj/project.pbxproj @@ -117,6 +117,8 @@ D0A564FE168D23D800AF6161 /* man in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; D0A56501168D258300AF6161 /* man in Copy Files */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; }; + D0C9733818DE5449002D7C81 /* utf8.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C9733718DE5449002D7C81 /* utf8.cpp */; }; + D0C9733918DE5449002D7C81 /* utf8.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C9733718DE5449002D7C81 /* utf8.cpp */; }; D0CBD587159EF0E10024809C /* launch_fish.scpt in Resources */ = {isa = PBXBuildFile; fileRef = D0CBD586159EF0E10024809C /* launch_fish.scpt */; }; D0D02A67159837AD008E62BD /* complete.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853713B3ACEE0099B651 /* complete.cpp */; }; D0D02A69159837B2008E62BD /* env.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853A13B3ACEE0099B651 /* env.cpp */; }; @@ -475,6 +477,8 @@ D0C6FCC914CFA4B0004CE8AD /* autoload.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = autoload.cpp; sourceTree = ""; }; D0C6FCCB14CFA4B7004CE8AD /* autoload.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = autoload.h; sourceTree = ""; }; D0C861EA16CC7054003B5A04 /* builtin_set_color.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_set_color.cpp; sourceTree = ""; }; + D0C9733718DE5449002D7C81 /* utf8.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = utf8.cpp; sourceTree = ""; }; + D0C9733A18DE5451002D7C81 /* utf8.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = utf8.h; sourceTree = ""; }; D0CA63F316FC275F00093BD4 /* builtin_printf.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_printf.cpp; sourceTree = ""; }; D0CBD580159EE48F0024809C /* config.fish */ = {isa = PBXFileReference; lastKnownFileType = text; name = config.fish; path = share/config.fish; sourceTree = ""; }; D0CBD583159EEE010024809C /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = System/Library/Frameworks/Foundation.framework; sourceTree = SDKROOT; }; @@ -686,6 +690,8 @@ D0A0855C13B3ACEE0099B651 /* signal.cpp */, D0A0852513B3ACEE0099B651 /* tokenizer.h */, D0A0855D13B3ACEE0099B651 /* tokenizer.cpp */, + D0C9733A18DE5451002D7C81 /* utf8.h */, + D0C9733718DE5449002D7C81 /* utf8.cpp */, D0A0852613B3ACEE0099B651 /* util.h */, D0A0855E13B3ACEE0099B651 /* util.cpp */, D0A0852713B3ACEE0099B651 /* wgetopt.h */, @@ -1120,6 +1126,7 @@ files = ( D0D02AC215985F3F008E62BD /* fishd.cpp in Sources */, D0D02AC315985F43008E62BD /* env_universal_common.cpp in Sources */, + D0C9733918DE5449002D7C81 /* utf8.cpp in Sources */, D0D02AC415985F4D008E62BD /* wutil.cpp in Sources */, D0D02AC515985F5B008E62BD /* print_help.cpp in Sources */, D0D02AC615985F65008E62BD /* common.cpp in Sources */, @@ -1157,6 +1164,7 @@ D0D02A86159839D5008E62BD /* postfork.cpp in Sources */, D0D02A87159839D5008E62BD /* screen.cpp in Sources */, D0D02A88159839D5008E62BD /* signal.cpp in Sources */, + D0C9733818DE5449002D7C81 /* utf8.cpp in Sources */, D0D2694A15983779005D9B9C /* builtin.cpp in Sources */, D0D2694915983772005D9B9C /* function.cpp in Sources */, D0D02A67159837AD008E62BD /* complete.cpp in Sources */, diff --git a/fish_tests.cpp b/fish_tests.cpp index 5b7bcea6d..f07b568c6 100644 --- a/fish_tests.cpp +++ b/fish_tests.cpp @@ -62,6 +62,7 @@ #include "parse_util.h" #include "pager.h" #include "input.h" +#include "utf8.h" static const char * const * s_arguments; static int s_test_run_count = 0; @@ -140,17 +141,17 @@ static void err(const wchar_t *blah, ...) va_list va; va_start(va, blah); err_count++; - + // show errors in red fputs("\x1b[31m", stdout); wprintf(L"Error: "); vwprintf(blah, va); va_end(va); - + // return to normal color fputs("\x1b[0m", stdout); - + wprintf(L"\n"); } @@ -857,6 +858,260 @@ static void test_utils() if (begin != a + wcslen(L"echo (echo (")) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__); } +/* UTF8 tests taken from Alexey Vatchenko's utf8 library. See http://www.bsdua.org/libbsdua.html */ + +static void test_utf82wchar(const char *src, size_t slen, const wchar_t *dst, size_t dlen, + int flags, size_t res, const char *descr) +{ + size_t size; + wchar_t *mem = NULL; + + /* Hack: if wchar is only UCS-2, and the UTF-8 input string contains astral characters, then tweak the expected size to 0 */ + if (src != NULL && is_wchar_ucs2()) + { + /* A UTF-8 code unit may represent an astral code point if it has 4 or more leading 1s */ + const unsigned char astral_mask = 0xF0; + for (size_t i=0; i < slen; i++) + { + if ((src[i] & astral_mask) == astral_mask) + { + /* Astral char. We expect this conversion to just fail. */ + res = 0; + break; + } + } + } + + if (dst != NULL) + { + mem = (wchar_t *)malloc(dlen * sizeof(*mem)); + if (mem == NULL) + { + err(L"u2w: %s: MALLOC FAILED\n", descr); + return; + } + } + + do + { + size = utf8_to_wchar(src, slen, mem, dlen, flags); + if (res != size) + { + err(L"u2w: %s: FAILED (rv: %lu, must be %lu)", descr, size, res); + break; + } + + if (mem == NULL) + break; /* OK */ + + if (memcmp(mem, dst, size * sizeof(*mem)) != 0) + { + err(L"u2w: %s: BROKEN", descr); + break; + } + + } + while (0); + + free(mem); +} + +static void test_wchar2utf8(const wchar_t *src, size_t slen, const char *dst, size_t dlen, + int flags, size_t res, const char *descr) +{ + size_t size; + char *mem = NULL; + + /* Hack: if wchar is simulating UCS-2, and the wchar_t input string contains astral characters, then tweak the expected size to 0 */ + if (src != NULL && is_wchar_ucs2()) + { + const uint32_t astral_mask = 0xFFFF0000U; + for (size_t i=0; i < slen; i++) + { + if ((src[i] & astral_mask) != 0) + { + /* astral char */ + res = 0; + break; + } + } + } + + if (dst != NULL) + { + mem = (char *)malloc(dlen); + if (mem == NULL) + { + err(L"w2u: %s: MALLOC FAILED", descr); + return; + } + } + + do + { + size = wchar_to_utf8(src, slen, mem, dlen, flags); + if (res != size) + { + err(L"w2u: %s: FAILED (rv: %lu, must be %lu)", descr, size, res); + break; + } + + if (mem == NULL) + break; /* OK */ + + if (memcmp(mem, dst, size) != 0) + { + err(L"w2u: %s: BROKEN", descr); + break; + } + + } + while (0); + + if (mem != NULL); + free(mem); +} + +static void test_utf8() +{ + wchar_t w1[] = {0x54, 0x65, 0x73, 0x74}; + wchar_t w2[] = {0x0422, 0x0435, 0x0441, 0x0442}; + wchar_t w3[] = {0x800, 0x1e80, 0x98c4, 0x9910, 0xff00}; + wchar_t w4[] = {0x15555, 0xf7777, 0xa}; + wchar_t w5[] = {0x255555, 0x1fa04ff, 0xddfd04, 0xa}; + wchar_t w6[] = {0xf255555, 0x1dfa04ff, 0x7fddfd04, 0xa}; + wchar_t wb[] = {-2, 0xa, 0xffffffff, 0x0441}; + wchar_t wm[] = {0x41, 0x0441, 0x3042, 0xff67, 0x9b0d, 0x2e05da67}; + wchar_t wb1[] = {0xa, 0x0422}; + wchar_t wb2[] = {0xd800, 0xda00, 0x41, 0xdfff, 0xa}; + wchar_t wbom[] = {0xfeff, 0x41, 0xa}; + wchar_t wbom2[] = {0x41, 0xa}; + wchar_t wbom22[] = {0xfeff, 0x41, 0xa}; + char u1[] = {0x54, 0x65, 0x73, 0x74}; + char u2[] = {0xd0, 0xa2, 0xd0, 0xb5, 0xd1, 0x81, 0xd1, 0x82}; + char u3[] = {0xe0, 0xa0, 0x80, 0xe1, 0xba, 0x80, 0xe9, 0xa3, 0x84, + 0xe9, 0xa4, 0x90, 0xef, 0xbc, 0x80 + }; + char u4[] = {0xf0, 0x95, 0x95, 0x95, 0xf3, 0xb7, 0x9d, 0xb7, 0xa}; + char u5[] = {0xf8, 0x89, 0x95, 0x95, 0x95, 0xf9, 0xbe, 0xa0, 0x93, + 0xbf, 0xf8, 0xb7, 0x9f, 0xb4, 0x84, 0x0a + }; + char u6[] = {0xfc, 0x8f, 0x89, 0x95, 0x95, 0x95, 0xfc, 0x9d, 0xbe, + 0xa0, 0x93, 0xbf, 0xfd, 0xbf, 0xb7, 0x9f, 0xb4, 0x84, 0x0a + }; + char ub[] = {0xa, 0xd1, 0x81}; + char um[] = {0x41, 0xd1, 0x81, 0xe3, 0x81, 0x82, 0xef, 0xbd, 0xa7, + 0xe9, 0xac, 0x8d, 0xfc, 0xae, 0x81, 0x9d, 0xa9, 0xa7 + }; + char ub1[] = {0xa, 0xff, 0xd0, 0xa2, 0xfe, 0x8f, 0xe0, 0x80}; + char uc080[] = {0xc0, 0x80}; + char ub2[] = {0xed, 0xa1, 0x8c, 0xed, 0xbe, 0xb4, 0xa}; + char ubom[] = {0x41, 0xa}; + char ubom2[] = {0xef, 0xbb, 0xbf, 0x41, 0xa}; + + /* + * UTF-8 -> UCS-4 string. + */ + test_utf82wchar(ubom2, sizeof(ubom2), wbom2, + sizeof(wbom2) / sizeof(*wbom2), UTF8_SKIP_BOM, + sizeof(wbom2) / sizeof(*wbom2), "skip BOM"); + test_utf82wchar(ubom2, sizeof(ubom2), wbom22, + sizeof(wbom22) / sizeof(*wbom22), 0, + sizeof(wbom22) / sizeof(*wbom22), "BOM"); + test_utf82wchar(uc080, sizeof(uc080), NULL, 0, 0, 0, + "c0 80 - forbitten by rfc3629"); + test_utf82wchar(ub2, sizeof(ub2), NULL, 0, 0, is_wchar_ucs2() ? 0 : 3, + "resulted in forbitten wchars (len)"); + test_utf82wchar(ub2, sizeof(ub2), wb2, sizeof(wb2) / sizeof(*wb2), 0, 0, + "resulted in forbitten wchars"); + test_utf82wchar(ub2, sizeof(ub2), L"\x0a", 1, UTF8_IGNORE_ERROR, + 1, "resulted in ignored forbitten wchars"); + test_utf82wchar(u1, sizeof(u1), w1, sizeof(w1) / sizeof(*w1), 0, + sizeof(w1) / sizeof(*w1), "1 octet chars"); + test_utf82wchar(u2, sizeof(u2), w2, sizeof(w2) / sizeof(*w2), 0, + sizeof(w2) / sizeof(*w2), "2 octets chars"); + test_utf82wchar(u3, sizeof(u3), w3, sizeof(w3) / sizeof(*w3), 0, + sizeof(w3) / sizeof(*w3), "3 octets chars"); + test_utf82wchar(u4, sizeof(u4), w4, sizeof(w4) / sizeof(*w4), 0, + sizeof(w4) / sizeof(*w4), "4 octets chars"); + test_utf82wchar(u5, sizeof(u5), w5, sizeof(w5) / sizeof(*w5), 0, + sizeof(w5) / sizeof(*w5), "5 octets chars"); + test_utf82wchar(u6, sizeof(u6), w6, sizeof(w6) / sizeof(*w6), 0, + sizeof(w6) / sizeof(*w6), "6 octets chars"); + test_utf82wchar("\xff", 1, NULL, 0, 0, 0, "broken utf-8 0xff symbol"); + test_utf82wchar("\xfe", 1, NULL, 0, 0, 0, "broken utf-8 0xfe symbol"); + test_utf82wchar("\x8f", 1, NULL, 0, 0, 0, + "broken utf-8, start from 10 higher bits"); + if (! is_wchar_ucs2()) test_utf82wchar(ub1, sizeof(ub1), wb1, sizeof(wb1) / sizeof(*wb1), + UTF8_IGNORE_ERROR, sizeof(wb1) / sizeof(*wb1), "ignore bad chars"); + test_utf82wchar(um, sizeof(um), wm, sizeof(wm) / sizeof(*wm), 0, + sizeof(wm) / sizeof(*wm), "mixed languages"); + test_utf82wchar(um, sizeof(um), wm, sizeof(wm) / sizeof(*wm) - 1, 0, + 0, "boundaries -1"); + test_utf82wchar(um, sizeof(um), wm, sizeof(wm) / sizeof(*wm) + 1, 0, + sizeof(wm) / sizeof(*wm), "boundaries +1"); + test_utf82wchar(um, sizeof(um), NULL, 0, 0, + sizeof(wm) / sizeof(*wm), "calculate length"); + test_utf82wchar(ub1, sizeof(ub1), NULL, 0, 0, + 0, "calculate length of bad chars"); + test_utf82wchar(ub1, sizeof(ub1), NULL, 0, + UTF8_IGNORE_ERROR, sizeof(wb1) / sizeof(*wb1), + "calculate length, ignore bad chars"); + test_utf82wchar(NULL, 0, NULL, 0, 0, 0, "invalid params, all 0"); + test_utf82wchar(u1, 0, NULL, 0, 0, 0, + "invalid params, src buf not NULL"); + test_utf82wchar(NULL, 10, NULL, 0, 0, 0, + "invalid params, src length is not 0"); + test_utf82wchar(u1, sizeof(u1), w1, 0, 0, 0, + "invalid params, dst is not NULL"); + + /* + * UCS-4 -> UTF-8 string. + */ + test_wchar2utf8(wbom, sizeof(wbom) / sizeof(*wbom), ubom, sizeof(ubom), + UTF8_SKIP_BOM, sizeof(ubom), "BOM"); + test_wchar2utf8(wb2, sizeof(wb2) / sizeof(*wb2), NULL, 0, 0, + 0, "prohibited wchars"); + test_wchar2utf8(wb2, sizeof(wb2) / sizeof(*wb2), NULL, 0, + UTF8_IGNORE_ERROR, 2, "ignore prohibited wchars"); + test_wchar2utf8(w1, sizeof(w1) / sizeof(*w1), u1, sizeof(u1), 0, + sizeof(u1), "1 octet chars"); + test_wchar2utf8(w2, sizeof(w2) / sizeof(*w2), u2, sizeof(u2), 0, + sizeof(u2), "2 octets chars"); + test_wchar2utf8(w3, sizeof(w3) / sizeof(*w3), u3, sizeof(u3), 0, + sizeof(u3), "3 octets chars"); + test_wchar2utf8(w4, sizeof(w4) / sizeof(*w4), u4, sizeof(u4), 0, + sizeof(u4), "4 octets chars"); + test_wchar2utf8(w5, sizeof(w5) / sizeof(*w5), u5, sizeof(u5), 0, + sizeof(u5), "5 octets chars"); + test_wchar2utf8(w6, sizeof(w6) / sizeof(*w6), u6, sizeof(u6), 0, + sizeof(u6), "6 octets chars"); + test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), ub, sizeof(ub), 0, + 0, "bad chars"); + test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), ub, sizeof(ub), + UTF8_IGNORE_ERROR, sizeof(ub), "ignore bad chars"); + test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), um, sizeof(um), 0, + sizeof(um), "mixed languages"); + test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), um, sizeof(um) - 1, 0, + 0, "boundaries -1"); + test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), um, sizeof(um) + 1, 0, + sizeof(um), "boundaries +1"); + test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), NULL, 0, 0, + sizeof(um), "calculate length"); + test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), NULL, 0, 0, + 0, "calculate length of bad chars"); + test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), NULL, 0, + UTF8_IGNORE_ERROR, sizeof(ub), + "calculate length, ignore bad chars"); + test_wchar2utf8(NULL, 0, NULL, 0, 0, 0, "invalid params, all 0"); + test_wchar2utf8(w1, 0, NULL, 0, 0, 0, + "invalid params, src buf not NULL"); + test_wchar2utf8(NULL, 10, NULL, 0, 0, 0, + "invalid params, src length is not 0"); + test_wchar2utf8(w1, sizeof(w1) / sizeof(*w1), u1, 0, 0, 0, + "invalid params, dst is not NULL"); +} + static void test_escape_sequences(void) { say(L"Testing escape codes"); @@ -1110,9 +1365,9 @@ static void test_path() static void test_pager_navigation() { say(L"Testing pager navigation"); - + /* Generate 19 strings of width 10. There's 2 spaces between completions, and our term size is 80; these can therefore fit into 6 columns (6 * 12 - 2 = 70) or 5 columns (58) but not 7 columns (7 * 12 - 2 = 82). - + You can simulate this test by creating 19 files named "file00.txt" through "file_18.txt". */ completion_list_t completions; @@ -1120,31 +1375,31 @@ static void test_pager_navigation() { append_completion(completions, L"abcdefghij"); } - + pager_t pager; pager.set_completions(completions); pager.set_term_size(80, 24); page_rendering_t render = pager.render(); - + if (render.term_width != 80) err(L"Wrong term width"); if (render.term_height != 24) err(L"Wrong term height"); - + size_t rows = 4, cols = 5; - + /* We have 19 completions. We can fit into 6 columns with 4 rows or 5 columns with 4 rows; the second one is better and so is what we ought to have picked. */ if (render.rows != rows) err(L"Wrong row count"); if (render.cols != cols) err(L"Wrong column count"); - + /* Initially expect to have no completion index */ if (render.selected_completion_idx != (size_t)(-1)) { err(L"Wrong initial selection"); } - + /* Here are navigation directions and where we expect the selection to be */ const struct { @@ -1155,31 +1410,31 @@ static void test_pager_navigation() { /* Tab completion to get into the list */ {direction_next, 0}, - + /* Westward motion in upper left wraps along the top row */ {direction_west, 16}, {direction_east, 1}, - + /* "Next" motion goes down the column */ {direction_next, 2}, {direction_next, 3}, - + {direction_west, 18}, {direction_east, 3}, {direction_east, 7}, {direction_east, 11}, {direction_east, 15}, {direction_east, 3}, - + {direction_west, 18}, {direction_east, 3}, - + /* Eastward motion wraps along the bottom, westward goes to the prior column */ {direction_east, 7}, {direction_east, 11}, {direction_east, 15}, {direction_east, 3}, - + /* Column memory */ {direction_west, 18}, {direction_south, 15}, @@ -1197,7 +1452,7 @@ static void test_pager_navigation() err(L"For command %lu, expected selection %lu, but found instead %lu\n", i, cmds[i].sel, render.selected_completion_idx); } } - + } enum word_motion_t @@ -1536,14 +1791,14 @@ static void test_complete(void) completions.clear(); complete(L"echo (builtin scuttlebut", completions, COMPLETION_REQUEST_DEFAULT); do_test(completions.size() == 0); - + /* Trailing spaces (#1261) */ complete_add(L"foobarbaz", false, 0, NULL, 0, NO_FILES, NULL, L"qux", NULL, COMPLETE_AUTO_SPACE); completions.clear(); complete(L"foobarbaz ", completions, COMPLETION_REQUEST_DEFAULT); do_test(completions.size() == 1); do_test(completions.at(0).completion == L"qux"); - + /* Don't complete variable names in single quotes (#1023) */ completions.clear(); complete(L"echo '$Foo", completions, COMPLETION_REQUEST_DEFAULT); @@ -1814,14 +2069,14 @@ static void test_input() wcstring desired_binding = prefix_binding + L'a'; input_mapping_add(prefix_binding.c_str(), L"up-line"); input_mapping_add(desired_binding.c_str(), L"down-line"); - + /* Push the desired binding on the stack (backwards!) */ size_t idx = desired_binding.size(); while (idx--) { input_unreadch(desired_binding.at(idx)); } - + /* Now test */ wint_t c = input_readch(); if (c != R_DOWN_LINE) @@ -2748,7 +3003,7 @@ static void test_highlighting(void) {L"'single_quote", highlight_spec_error}, {NULL, -1} }; - + const highlight_component_t components11[] = { {L"echo", highlight_spec_command}, @@ -2761,7 +3016,7 @@ static void test_highlighting(void) {L"]", highlight_spec_operator}, {NULL, -1} }; - + const highlight_component_t components12[] = { {L"for", highlight_spec_command}, @@ -2867,6 +3122,7 @@ int main(int argc, char **argv) if (should_test_function("cancellation")) test_cancellation(); if (should_test_function("indents")) test_indents(); if (should_test_function("utils")) test_utils(); + if (should_test_function("utf8")) test_utf8(); if (should_test_function("escape_sequences")) test_escape_sequences(); if (should_test_function("lru")) test_lru(); if (should_test_function("expand")) test_expand(); @@ -2906,7 +3162,8 @@ int main(int argc, char **argv) event_destroy(); proc_destroy(); - if(err_count != 0) { + if (err_count != 0) + { return(1); } } diff --git a/utf8.cpp b/utf8.cpp new file mode 100644 index 000000000..60d83d33d --- /dev/null +++ b/utf8.cpp @@ -0,0 +1,514 @@ +/* + * Copyright (c) 2007 Alexey Vatchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include + +#include + +#include "utf8.h" + +#include +#include +#include + + +#define _NXT 0x80 +#define _SEQ2 0xc0 +#define _SEQ3 0xe0 +#define _SEQ4 0xf0 +#define _SEQ5 0xf8 +#define _SEQ6 0xfc + +#define _BOM 0xfeff + +/* We can tweak the following typedef to allow us to simulate Windows-style 16 bit wchar's on Unix */ +typedef wchar_t utf8_wchar_t; +#define UTF8_WCHAR_MAX ((size_t)std::numeric_limits::max()) + +bool is_wchar_ucs2() +{ + return UTF8_WCHAR_MAX <= 0xFFFF; +} + +static size_t utf8_to_wchar_internal(const char *in, size_t insize, utf8_wchar_t *out, size_t outsize, int flags); +static size_t wchar_to_utf8_internal(const utf8_wchar_t *in, size_t insize, char *out, size_t outsize, int flags); + +static bool safe_copy_wchar_to_utf8_wchar(const wchar_t *in, utf8_wchar_t *out, size_t count) +{ + bool result = true; + for (size_t i=0; i < count; i++) + { + wchar_t c = in[i]; + if (c > UTF8_WCHAR_MAX) + { + result = false; + break; + } + out[i] = c; + } + return result; +} + +bool utf8_to_wchar_string(const std::string &str, std::wstring *result) +{ + result->clear(); + const size_t inlen = str.size(); + if (inlen == 0) + { + return true; + } + + bool success = false; + const char *input = str.c_str(); + size_t outlen = utf8_to_wchar(input, inlen, NULL, 0, 0); + if (outlen > 0) + { + wchar_t *tmp = new wchar_t[outlen]; + size_t outlen2 = utf8_to_wchar(input, inlen, tmp, outlen, 0); + if (outlen2 > 0) + { + result->assign(tmp, outlen2); + success = true; + } + delete[] tmp; + } + return success; +} + +bool wchar_to_utf8_string(const std::wstring &str, std::string *result) +{ + result->clear(); + const size_t inlen = str.size(); + if (inlen == 0) + { + return true; + } + + bool success = false; + const wchar_t *input = str.c_str(); + size_t outlen = wchar_to_utf8(input, inlen, NULL, 0, 0); + if (outlen > 0) + { + char *tmp = new char[outlen]; + size_t outlen2 = wchar_to_utf8(input, inlen, tmp, outlen, 0); + if (outlen2 > 0) + { + result->assign(tmp, outlen2); + success = true; + } + delete[] tmp; + } + return success; +} + +size_t utf8_to_wchar(const char *in, size_t insize, wchar_t *out, size_t outsize, int flags) +{ + if (in == NULL || insize == 0 || (outsize == 0 && out != NULL)) + { + return 0; + } + + size_t result; + if (sizeof(wchar_t) == sizeof(utf8_wchar_t)) + { + result = utf8_to_wchar_internal(in, insize, reinterpret_cast(out), outsize, flags); + } + else + { + // Allocate a temporary buffer to hold the output + // note: outsize may be 0 + utf8_wchar_t *tmp_output = new utf8_wchar_t[outsize]; + + // Invoke the conversion with the temporary + result = utf8_to_wchar_internal(in, insize, tmp_output, outsize, flags); + + // Copy back from tmp to the function's output, then clean it up + size_t amount_to_copy = std::min(result, outsize); + std::copy(tmp_output, tmp_output + amount_to_copy, out); + delete[] tmp_output; + } + return result; +} + +size_t wchar_to_utf8(const wchar_t *in, size_t insize, char *out, size_t outsize, int flags) +{ + if (in == NULL || insize == 0 || (outsize == 0 && out != NULL)) + { + return 0; + } + + size_t result; + if (sizeof(wchar_t) == sizeof(utf8_wchar_t)) + { + result = wchar_to_utf8_internal(reinterpret_cast(in), insize, out, outsize, flags); + } + else + { + // Allocate a temporary buffer to hold the input + // the std::copy performs the size conversion + // note: insize may be 0 + utf8_wchar_t *tmp_input = new utf8_wchar_t[insize]; + if (! safe_copy_wchar_to_utf8_wchar(in, tmp_input, insize)) + { + // our utf8_wchar_t is UCS-16 and there was an astral character + result = 0; + } + else + { + // Invoke the conversion with the temporary, then clean up the input + result = wchar_to_utf8_internal(tmp_input, insize, out, outsize, flags); + } + delete[] tmp_input; + } + return result; +} + + +static int __wchar_forbitten(utf8_wchar_t sym); +static int __utf8_forbitten(unsigned char octet); + +static int +__wchar_forbitten(utf8_wchar_t sym) +{ + + /* Surrogate pairs */ + if (sym >= 0xd800 && sym <= 0xdfff) + return (-1); + + return (0); +} + +static int +__utf8_forbitten(unsigned char octet) +{ + + switch (octet) + { + case 0xc0: + case 0xc1: + case 0xf5: + case 0xff: + return (-1); + } + + return (0); +} + +/* + * DESCRIPTION + * This function translates UTF-8 string into UCS-2 or UCS-4 string (all symbols + * will be in local machine byte order). + * + * It takes the following arguments: + * in - input UTF-8 string. It can be null-terminated. + * insize - size of input string in bytes. + * out - result buffer for UCS-2/4 string. If out is NULL, + * function returns size of result buffer. + * outsize - size of out buffer in wide characters. + * + * RETURN VALUES + * The function returns size of result buffer (in wide characters). + * Zero is returned in case of error. + * + * CAVEATS + * 1. If UTF-8 string contains zero symbols, they will be translated + * as regular symbols. + * 2. If UTF8_IGNORE_ERROR or UTF8_SKIP_BOM flag is set, sizes may vary + * when `out' is NULL and not NULL. It's because of special UTF-8 + * sequences which may result in forbitten (by RFC3629) UNICODE + * characters. So, the caller must check return value every time and + * not prepare buffer in advance (\0 terminate) but after calling this + * function. + */ +static size_t utf8_to_wchar_internal(const char *in, size_t insize, utf8_wchar_t *out, size_t outsize, int flags) +{ + unsigned char *p, *lim; + utf8_wchar_t *wlim, high; + size_t n, total, i, n_bits; + + if (in == NULL || insize == 0 || (outsize == 0 && out != NULL)) + return (0); + + total = 0; + p = (unsigned char *)in; + lim = p + insize; + wlim = out + outsize; + + for (; p < lim; p += n) + { + if (__utf8_forbitten(*p) != 0 && + (flags & UTF8_IGNORE_ERROR) == 0) + return (0); + + /* + * Get number of bytes for one wide character. + */ + n = 1; /* default: 1 byte. Used when skipping bytes. */ + if ((*p & 0x80) == 0) + high = (utf8_wchar_t)*p; + else if ((*p & 0xe0) == _SEQ2) + { + n = 2; + high = (utf8_wchar_t)(*p & 0x1f); + } + else if ((*p & 0xf0) == _SEQ3) + { + n = 3; + high = (utf8_wchar_t)(*p & 0x0f); + } + else if ((*p & 0xf8) == _SEQ4) + { + n = 4; + high = (utf8_wchar_t)(*p & 0x07); + } + else if ((*p & 0xfc) == _SEQ5) + { + n = 5; + high = (utf8_wchar_t)(*p & 0x03); + } + else if ((*p & 0xfe) == _SEQ6) + { + n = 6; + high = (utf8_wchar_t)(*p & 0x01); + } + else + { + if ((flags & UTF8_IGNORE_ERROR) == 0) + return (0); + continue; + } + + /* does the sequence header tell us truth about length? */ + if (lim - p <= n - 1) + { + if ((flags & UTF8_IGNORE_ERROR) == 0) + return (0); + n = 1; + continue; /* skip */ + } + + /* + * Validate sequence. + * All symbols must have higher bits set to 10xxxxxx + */ + if (n > 1) + { + for (i = 1; i < n; i++) + { + if ((p[i] & 0xc0) != _NXT) + break; + } + if (i != n) + { + if ((flags & UTF8_IGNORE_ERROR) == 0) + return (0); + n = 1; + continue; /* skip */ + } + } + + total++; + + if (out == NULL) + continue; + + if (out >= wlim) + return (0); /* no space left */ + + uint32_t out_val = 0; + *out = 0; + n_bits = 0; + for (i = 1; i < n; i++) + { + out_val |= (utf8_wchar_t)(p[n - i] & 0x3f) << n_bits; + n_bits += 6; /* 6 low bits in every byte */ + } + out_val |= high << n_bits; + + bool skip = false; + if (__wchar_forbitten(out_val) != 0) + { + if ((flags & UTF8_IGNORE_ERROR) == 0) + { + return 0; /* forbitten character */ + } + else + { + skip = true; + } + } + else if (out_val == _BOM && (flags & UTF8_SKIP_BOM) != 0) + { + skip = true; + } + + if (skip) + { + total--; + } + else if (out_val > UTF8_WCHAR_MAX) + { + // wchar_t is UCS-2, but the UTF-8 specified an astral character + return 0; + } + else + { + *out++ = out_val; + } + } + + return (total); +} + +/* + * DESCRIPTION + * This function translates UCS-2/4 symbols (given in local machine + * byte order) into UTF-8 string. + * + * It takes the following arguments: + * in - input unicode string. It can be null-terminated. + * insize - size of input string in wide characters. + * out - result buffer for utf8 string. If out is NULL, + * function returns size of result buffer. + * outsize - size of result buffer. + * + * RETURN VALUES + * The function returns size of result buffer (in bytes). Zero is returned + * in case of error. + * + * CAVEATS + * If UCS-4 string contains zero symbols, they will be translated + * as regular symbols. + */ +static size_t wchar_to_utf8_internal(const utf8_wchar_t *in, size_t insize, char *out, size_t outsize, int flags) +{ + const utf8_wchar_t *w, *wlim; + unsigned char *p, *lim; + size_t total, n; + + if (in == NULL || insize == 0 || (outsize == 0 && out != NULL)) + return (0); + + w = in; + wlim = w + insize; + p = (unsigned char *)out; + lim = p + outsize; + total = 0; + for (; w < wlim; w++) + { + if (__wchar_forbitten(*w) != 0) + { + if ((flags & UTF8_IGNORE_ERROR) == 0) + return (0); + else + continue; + } + + if (*w == _BOM && (flags & UTF8_SKIP_BOM) != 0) + continue; + + const int32_t w_wide = *w; + if (w_wide < 0) + { + if ((flags & UTF8_IGNORE_ERROR) == 0) + return (0); + continue; + } + else if (w_wide <= 0x0000007f) + n = 1; + else if (w_wide <= 0x000007ff) + n = 2; + else if (w_wide <= 0x0000ffff) + n = 3; + else if (w_wide <= 0x001fffff) + n = 4; + else if (w_wide <= 0x03ffffff) + n = 5; + else /* if (w_wide <= 0x7fffffff) */ + n = 6; + + total += n; + + if (out == NULL) + continue; + + if (lim - p <= n - 1) + return (0); /* no space left */ + + /* extract the wchar_t as big-endian. If wchar_t is UCS-16, the first two bytes will be 0 */ + unsigned char oc[4]; + uint32_t w_tmp = *w; + oc[3] = w_tmp & 0xFF; + w_tmp >>= 8; + oc[2] = w_tmp & 0xFF; + w_tmp >>= 8; + oc[1] = w_tmp & 0xFF; + w_tmp >>= 8; + oc[0] = w_tmp & 0xFF; + + switch (n) + { + case 1: + p[0] = oc[3]; + break; + + case 2: + p[1] = _NXT | (oc[3] & 0x3f); + p[0] = _SEQ2 | (oc[3] >> 6) | ((oc[2] & 0x07) << 2); + break; + + case 3: + p[2] = _NXT | (oc[3] & 0x3f); + p[1] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2); + p[0] = _SEQ3 | ((oc[2] & 0xf0) >> 4); + break; + + case 4: + p[3] = _NXT | (oc[3] & 0x3f); + p[2] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2); + p[1] = _NXT | ((oc[2] & 0xf0) >> 4) | + ((oc[1] & 0x03) << 4); + p[0] = _SEQ4 | ((oc[1] & 0x1f) >> 2); + break; + + case 5: + p[4] = _NXT | (oc[3] & 0x3f); + p[3] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2); + p[2] = _NXT | ((oc[2] & 0xf0) >> 4) | + ((oc[1] & 0x03) << 4); + p[1] = _NXT | (oc[1] >> 2); + p[0] = _SEQ5 | (oc[0] & 0x03); + break; + + case 6: + p[5] = _NXT | (oc[3] & 0x3f); + p[4] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2); + p[3] = _NXT | (oc[2] >> 4) | ((oc[1] & 0x03) << 4); + p[2] = _NXT | (oc[1] >> 2); + p[1] = _NXT | (oc[0] & 0x3f); + p[0] = _SEQ6 | ((oc[0] & 0x40) >> 6); + break; + } + + /* + * NOTE: do not check here for forbitten UTF-8 characters. + * They cannot appear here because we do proper convertion. + */ + + p += n; + } + + return (total); +} diff --git a/utf8.h b/utf8.h new file mode 100644 index 000000000..18aa52658 --- /dev/null +++ b/utf8.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2007 Alexey Vatchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/* + * utf8: implementation of UTF-8 charset encoding (RFC3629). + */ +#ifndef _UTF8_H_ +#define _UTF8_H_ + +#include + +#include +#include + +#define UTF8_IGNORE_ERROR 0x01 +#define UTF8_SKIP_BOM 0x02 + +bool utf8_to_wchar_string(const std::string &input, std::wstring *result); +bool wchar_to_utf8_string(const std::wstring &input, std::string *result); + +size_t utf8_to_wchar(const char *in, size_t insize, wchar_t *out, size_t outsize, int flags); +size_t wchar_to_utf8(const wchar_t *in, size_t insize, char *out, size_t outsize, int flags); + +bool is_wchar_ucs2(); + +#endif /* !_UTF8_H_ */ From a67dd9fbdd475ce386f9d87f83e7a1c628777eff Mon Sep 17 00:00:00 2001 From: Siteshwar Vashisht Date: Sun, 23 Mar 2014 15:09:43 +0530 Subject: [PATCH 05/13] Included missing stdint.h header in utf8.cpp --- utf8.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utf8.cpp b/utf8.cpp index 60d83d33d..196b52681 100644 --- a/utf8.cpp +++ b/utf8.cpp @@ -13,8 +13,9 @@ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#include +#include +#include #include #include "utf8.h" @@ -23,7 +24,6 @@ #include #include - #define _NXT 0x80 #define _SEQ2 0xc0 #define _SEQ3 0xe0 From 9718e70260fd44ede9409711d7d0a8efcf131c20 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Sun, 23 Mar 2014 13:06:24 -0700 Subject: [PATCH 06/13] Naive reimplementation of utf2wcs and wcs2utf in env_universal_common.cpp. These use the new utf8 functions exposed in utf8.h. This will allow us to drop the iconv dependency. --- env_universal_common.cpp | 305 ++------------------------------------- utf8.h | 2 + 2 files changed, 17 insertions(+), 290 deletions(-) diff --git a/env_universal_common.cpp b/env_universal_common.cpp index c175b4a14..048396afa 100644 --- a/env_universal_common.cpp +++ b/env_universal_common.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include #include @@ -39,6 +38,7 @@ #include "common.h" #include "wutil.h" +#include "utf8.h" #include "env_universal_common.h" /** @@ -116,304 +116,29 @@ static void (*callback)(fish_message_type_t type, const wchar_t *key, const wchar_t *val); -/** - List of names for the UTF-8 character set. - */ -static const char *iconv_utf8_names[]= +/* UTF <-> wchar conversions. These return a string allocated with malloc. These call sites could be cleaned up substantially to eliminate the dependence on malloc. */ +static wchar_t *utf2wcs(const char *input) { - "utf-8", "UTF-8", - "utf8", "UTF8", - 0 -} -; - -/** - List of wide character names, undefined byte length. - */ -static const char *iconv_wide_names_unknown[]= -{ - "wchar_t", "WCHAR_T", - "wchar", "WCHAR", - 0 -} -; - -/** - List of wide character names, 4 bytes long. - */ -static const char *iconv_wide_names_4[]= -{ - "wchar_t", "WCHAR_T", - "wchar", "WCHAR", - "ucs-4", "UCS-4", - "ucs4", "UCS4", - "utf-32", "UTF-32", - "utf32", "UTF32", - 0 -} -; - -/** - List of wide character names, 2 bytes long. - */ -static const char *iconv_wide_names_2[]= -{ - "wchar_t", "WCHAR_T", - "wchar", "WCHAR", - "ucs-2", "UCS-2", - "ucs2", "UCS2", - "utf-16", "UTF-16", - "utf16", "UTF16", - 0 -} -; - -template -class sloppy {}; - -static size_t hack_iconv(iconv_t cd, const char * const* inbuf, size_t *inbytesleft, char **outbuf, size_t *outbytesleft) -{ - /* FreeBSD has this prototype: size_t iconv (iconv_t, const char **...) - OS X and Linux this one: size_t iconv (iconv_t, char **...) - AFAIK there's no single type that can be passed as both char ** and const char **. - Therefore, we let C++ figure it out, by providing a struct with an implicit conversion to both char** and const char **. - */ - struct sloppy_char + wchar_t *result = NULL; + wcstring converted; + if (utf8_to_wchar_string(input, &converted)) { - const char * const * t; - operator char** () const - { - return (char **)t; - } - operator const char** () const - { - return (const char**)t; - } - } slop_inbuf = {inbuf}; - - return iconv(cd, slop_inbuf, inbytesleft, outbuf, outbytesleft); + result = wcsdup(converted.c_str()); + } + return result; } -/** - Convert utf-8 string to wide string - */ -static wchar_t *utf2wcs(const char *in) +static char *wcs2utf(const wchar_t *input) { - iconv_t cd=(iconv_t) -1; - int i,j; - - wchar_t *out; - - /* - Try to convert to wchar_t. If that is not a valid character set, - try various names for ucs-4. We can't be sure that ucs-4 is - really the character set used by wchar_t, but it is the best - assumption we can make. - */ - const char **to_name=0; - - switch (sizeof(wchar_t)) + char *result = NULL; + std::string converted; + if (wchar_to_utf8_string(input, &converted)) { - - case 2: - to_name = iconv_wide_names_2; - break; - - case 4: - to_name = iconv_wide_names_4; - break; - - default: - to_name = iconv_wide_names_unknown; - break; + result = strdup(converted.c_str()); } - - - /* - The line protocol fish uses is always utf-8. - */ - const char **from_name = iconv_utf8_names; - - size_t in_len = strlen(in); - size_t out_len = sizeof(wchar_t)*(in_len+2); - size_t nconv; - char *nout; - - out = (wchar_t *)malloc(out_len); - nout = (char *)out; - - if (!out) - return 0; - - for (i=0; to_name[i]; i++) - { - for (j=0; from_name[j]; j++) - { - cd = iconv_open(to_name[i], from_name[j]); - - if (cd != (iconv_t) -1) - { - goto start_conversion; - - } - } - } - -start_conversion: - - if (cd == (iconv_t) -1) - { - /* Something went wrong. */ - debug(0, L"Could not perform utf-8 conversion"); - if (errno != EINVAL) - wperror(L"iconv_open"); - - /* Terminate the output string. */ - free(out); - return 0; - } - - /* FreeBSD has this prototype: size_t iconv (iconv_t, const char **...) - OS X and Linux this one: size_t iconv (iconv_t, char **...) - AFAIK there's no single type that can be passed as both char ** and const char **. - Hence this hack. - */ - nconv = hack_iconv(cd, &in, &in_len, &nout, &out_len); - - if (nconv == (size_t) -1) - { - debug(0, L"Error while converting from utf string"); - return 0; - } - - *((wchar_t *) nout) = L'\0'; - - /* - Check for silly iconv behaviour inserting an bytemark in the output - string. - */ - if (*out == L'\xfeff' || *out == L'\xffef' || *out == L'\xefbbbf') - { - wchar_t *out_old = out; - out = wcsdup(out+1); - if (! out) - { - debug(0, L"FNORD!!!!"); - free(out_old); - return 0; - } - free(out_old); - } - - - if (iconv_close(cd) != 0) - wperror(L"iconv_close"); - - return out; + return result; } - - -/** - Convert wide string to utf-8 - */ -static char *wcs2utf(const wchar_t *in) -{ - iconv_t cd=(iconv_t) -1; - int i,j; - - char *char_in = (char *)in; - char *out; - - /* - Try to convert to wchar_t. If that is not a valid character set, - try various names for ucs-4. We can't be sure that ucs-4 is - really the character set used by wchar_t, but it is the best - assumption we can make. - */ - const char **from_name=0; - - switch (sizeof(wchar_t)) - { - - case 2: - from_name = iconv_wide_names_2; - break; - - case 4: - from_name = iconv_wide_names_4; - break; - - default: - from_name = iconv_wide_names_unknown; - break; - } - - const char **to_name = iconv_utf8_names; - - size_t in_len = wcslen(in); - size_t out_len = sizeof(char)*((MAX_UTF8_BYTES*in_len)+1); - size_t nconv; - char *nout; - - out = (char *)malloc(out_len); - nout = (char *)out; - in_len *= sizeof(wchar_t); - - if (!out) - return 0; - - for (i=0; to_name[i]; i++) - { - for (j=0; from_name[j]; j++) - { - cd = iconv_open(to_name[i], from_name[j]); - - if (cd != (iconv_t) -1) - { - goto start_conversion; - - } - } - } - -start_conversion: - - if (cd == (iconv_t) -1) - { - /* Something went wrong. */ - debug(0, L"Could not perform utf-8 conversion"); - if (errno != EINVAL) - wperror(L"iconv_open"); - - /* Terminate the output string. */ - free(out); - return 0; - } - - nconv = hack_iconv(cd, &char_in, &in_len, &nout, &out_len); - - - if (nconv == (size_t) -1) - { - debug(0, L"%d %d", in_len, out_len); - debug(0, L"Error while converting from to string"); - - /* Terminate the output string. */ - free(out); - return 0; - } - - *nout = '\0'; - - if (iconv_close(cd) != 0) - wperror(L"iconv_close"); - - return out; -} - - - void env_universal_common_init(void (*cb)(fish_message_type_t type, const wchar_t *key, const wchar_t *val)) { callback = cb; diff --git a/utf8.h b/utf8.h index 18aa52658..a1f501545 100644 --- a/utf8.h +++ b/utf8.h @@ -28,9 +28,11 @@ #define UTF8_IGNORE_ERROR 0x01 #define UTF8_SKIP_BOM 0x02 +/* Convert a string between UTF8 and UCS-2/4 (depending on size of wchar_t). Returns true if successful, storing the result of the conversion in *result */ bool utf8_to_wchar_string(const std::string &input, std::wstring *result); bool wchar_to_utf8_string(const std::wstring &input, std::string *result); +/* Variants exposed for testing */ size_t utf8_to_wchar(const char *in, size_t insize, wchar_t *out, size_t outsize, int flags); size_t wchar_to_utf8(const wchar_t *in, size_t insize, char *out, size_t outsize, int flags); From 0d64bbed46f07f78a24449139d11e7fafe968e4e Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Sun, 23 Mar 2014 13:29:57 -0700 Subject: [PATCH 07/13] Stop linking iconv, now that we have our own utf8 conversion routines --- configure.ac | 7 ------- fish.xcodeproj/project.pbxproj | 10 ---------- 2 files changed, 17 deletions(-) diff --git a/configure.ac b/configure.ac index b5ba9f0ba..f30187fbb 100644 --- a/configure.ac +++ b/configure.ac @@ -105,8 +105,6 @@ echo "CXXFLAGS: $CXXFLAGS" # # This mostly helps OS X users, since fink usually installs out of # tree and doesn't update CXXFLAGS. -# -# It also helps FreeBSD which puts libiconv in /usr/local/lib for i in /usr/pkg /sw /opt /opt/local /usr/local; do @@ -421,10 +419,6 @@ LIBS_SHARED=$LIBS # LIBS="$LIBS_SHARED" -# Check for libiconv_open if we can't find iconv_open. Silly OS X does -# weird macro magic for the sole purpose of amusing me. -AC_SEARCH_LIBS( iconv_open, iconv, , [AC_SEARCH_LIBS( libiconv_open, iconv, , [AC_MSG_ERROR([Could not find an iconv implementation, needed to build fish])] )] ) - LIBS_FISH=$LIBS # @@ -439,7 +433,6 @@ LIBS_FISH_INDENT=$LIBS # LIBS="$LIBS_SHARED" -AC_SEARCH_LIBS( iconv_open, iconv, , [AC_SEARCH_LIBS( libiconv_open, iconv, , [AC_MSG_ERROR([Could not find an iconv implementation, needed to build fish])] )] ) LIBS_FISHD=$LIBS # diff --git a/fish.xcodeproj/project.pbxproj b/fish.xcodeproj/project.pbxproj index 47d04c4cc..c7a6abaed 100644 --- a/fish.xcodeproj/project.pbxproj +++ b/fish.xcodeproj/project.pbxproj @@ -113,7 +113,6 @@ D08A32B917B446B100F3A533 /* parse_productions.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */; }; D08A32BA17B446B100F3A533 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; }; D08A32BC17B4473B00F3A533 /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; }; - D08A32BD17B4474000F3A533 /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; }; D0A564FE168D23D800AF6161 /* man in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; D0A56501168D258300AF6161 /* man in Copy Files */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; }; @@ -154,7 +153,6 @@ D0D02A87159839D5008E62BD /* screen.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855A13B3ACEE0099B651 /* screen.cpp */; }; D0D02A88159839D5008E62BD /* signal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855C13B3ACEE0099B651 /* signal.cpp */; }; D0D02A89159839DF008E62BD /* fish.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854213B3ACEE0099B651 /* fish.cpp */; }; - D0D02A8B15983CDF008E62BD /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; }; D0D02A8D15983CFA008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; }; D0D02A8F15983D8F008E62BD /* parser_keywords.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855313B3ACEE0099B651 /* parser_keywords.cpp */; }; D0D02AC215985F3F008E62BD /* fishd.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854313B3ACEE0099B651 /* fishd.cpp */; }; @@ -163,7 +161,6 @@ D0D02AC515985F5B008E62BD /* print_help.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855613B3ACEE0099B651 /* print_help.cpp */; }; D0D02AC615985F65008E62BD /* common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853613B3ACEE0099B651 /* common.cpp */; }; D0D02AC715985F9D008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; }; - D0D02AC815985F9F008E62BD /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; }; D0D02AD615986492008E62BD /* fish_indent.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853F13B3ACEE0099B651 /* fish_indent.cpp */; }; D0D02AD715986498008E62BD /* print_help.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855613B3ACEE0099B651 /* print_help.cpp */; }; D0D02AD81598649E008E62BD /* common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853613B3ACEE0099B651 /* common.cpp */; }; @@ -171,7 +168,6 @@ D0D02ADA159864AB008E62BD /* wutil.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0856113B3ACEE0099B651 /* wutil.cpp */; }; D0D02ADB159864C2008E62BD /* tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855D13B3ACEE0099B651 /* tokenizer.cpp */; }; D0D02ADC159864D5008E62BD /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; }; - D0D02ADD159864D7008E62BD /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; }; D0D2694915983772005D9B9C /* function.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854413B3ACEE0099B651 /* function.cpp */; }; D0D2694A15983779005D9B9C /* builtin.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853513B3ACEE0099B651 /* builtin.cpp */; }; D0F019F115A977140034B3B1 /* fish in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0D2693C159835CA005D9B9C /* fish */; }; @@ -483,7 +479,6 @@ D0CBD580159EE48F0024809C /* config.fish */ = {isa = PBXFileReference; lastKnownFileType = text; name = config.fish; path = share/config.fish; sourceTree = ""; }; D0CBD583159EEE010024809C /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = System/Library/Frameworks/Foundation.framework; sourceTree = SDKROOT; }; D0CBD586159EF0E10024809C /* launch_fish.scpt */ = {isa = PBXFileReference; lastKnownFileType = file; name = launch_fish.scpt; path = osx/launch_fish.scpt; sourceTree = ""; }; - D0D02A8A15983CDF008E62BD /* libiconv.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libiconv.dylib; path = usr/lib/libiconv.dylib; sourceTree = SDKROOT; }; D0D02A8C15983CFA008E62BD /* libncurses.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libncurses.dylib; path = usr/lib/libncurses.dylib; sourceTree = SDKROOT; }; D0D02A9A15985A75008E62BD /* fish.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = fish.app; sourceTree = BUILT_PRODUCTS_DIR; }; D0D02AA915985C0C008E62BD /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = Info.plist; path = osx/Info.plist; sourceTree = ""; }; @@ -503,7 +498,6 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - D08A32BD17B4474000F3A533 /* libiconv.dylib in Frameworks */, D08A32BC17B4473B00F3A533 /* libncurses.dylib in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; @@ -513,7 +507,6 @@ buildActionMask = 2147483647; files = ( D0D02AC715985F9D008E62BD /* libncurses.dylib in Frameworks */, - D0D02AC815985F9F008E62BD /* libiconv.dylib in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -522,7 +515,6 @@ buildActionMask = 2147483647; files = ( D0D02ADC159864D5008E62BD /* libncurses.dylib in Frameworks */, - D0D02ADD159864D7008E62BD /* libiconv.dylib in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -531,7 +523,6 @@ buildActionMask = 2147483647; files = ( D0D02A8D15983CFA008E62BD /* libncurses.dylib in Frameworks */, - D0D02A8B15983CDF008E62BD /* libiconv.dylib in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -587,7 +578,6 @@ isa = PBXGroup; children = ( D0D02A8C15983CFA008E62BD /* libncurses.dylib */, - D0D02A8A15983CDF008E62BD /* libiconv.dylib */, D0CBD583159EEE010024809C /* Foundation.framework */, ); name = Libraries; From a4cafaad2e2eabcf3262ae503d616d767c9d0f4d Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Tue, 25 Mar 2014 00:34:22 -0700 Subject: [PATCH 08/13] Turn on the new "ast" parser by default for execution. This change replaces fish's execution model, and obviates much of parser_t. Instead of parsing fish code into a sequence of commands-arguments, this reifies syntactic constructs into a grammar, builds a parse tree, and executes that. This provides a big simplification and (sometimes) performance boost. fish while loops become C++ while loops, etc. There are some known regressions in error reporting, which ought to be fixed in the soon-to-be-merged parser_cleanup branch. There's also legitimate changes in edge cases. For example, `command builtin ...` now executes a command called "builtin" instead of doing something else weird. The most significant change is that syntactic elements must be unexpected: for example, single quoting 'command' will now cause it to not be recognized. This should be fixed soon. Please open issues for any regressions you find! --- parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser.cpp b/parser.cpp index d512f77ef..077d78747 100644 --- a/parser.cpp +++ b/parser.cpp @@ -3145,7 +3145,7 @@ bool parser_use_ast(void) env_var_t var = env_get_string(L"fish_new_parser"); if (var.missing_or_empty()) { - return 0; + return 1; } else { From b520a03c57a510b978ac5ee3382b446b72107669 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Tue, 25 Mar 2014 12:44:21 -0700 Subject: [PATCH 09/13] Prefer swap() member function to std::swap. We were hitting the inefficient generic std::swap for some derived types. --- builtin_set_color.cpp | 5 ++--- highlight.cpp | 2 +- parse_execution.cpp | 2 +- parse_tree.cpp | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/builtin_set_color.cpp b/builtin_set_color.cpp index 6e8f55b0a..0fdc1ac0b 100644 --- a/builtin_set_color.cpp +++ b/builtin_set_color.cpp @@ -235,9 +235,8 @@ static int builtin_set_color(parser_t &parser, wchar_t **argv) output_set_writer(saved_writer_func); /* Output the collected string */ - std::string local_output; - std::swap(builtin_set_color_output, local_output); - stdout_buffer.append(str2wcstring(local_output)); + stdout_buffer.append(str2wcstring(builtin_set_color_output)); + builtin_set_color_output.clear(); return STATUS_BUILTIN_OK; } diff --git a/highlight.cpp b/highlight.cpp index eaaec750a..87985ae6b 100644 --- a/highlight.cpp +++ b/highlight.cpp @@ -350,7 +350,7 @@ bool plain_statement_get_expanded_command(const wcstring &src, const parse_node_ if (expand_one(cmd, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES | EXPAND_SKIP_JOBS)) { /* Success, return the expanded string by reference */ - std::swap(cmd, *out_cmd); + out_cmd->swap(cmd); result = true; } } diff --git a/parse_execution.cpp b/parse_execution.cpp index 15c7de5f2..9267fb79b 100644 --- a/parse_execution.cpp +++ b/parse_execution.cpp @@ -1090,7 +1090,7 @@ bool parse_execution_context_t::determine_io_chain(const parse_node_t &statement if (out_chain && ! errored) { - std::swap(*out_chain, result); + out_chain->swap(result); } return ! errored; } diff --git a/parse_tree.cpp b/parse_tree.cpp index ae8e42d02..a1de4f966 100644 --- a/parse_tree.cpp +++ b/parse_tree.cpp @@ -687,13 +687,13 @@ void parse_ll_t::acquire_output(parse_node_tree_t *output, parse_error_list_t *e { if (output != NULL) { - std::swap(*output, this->nodes); + output->swap(this->nodes); } this->nodes.clear(); if (errors != NULL) { - std::swap(*errors, this->errors); + errors->swap(this->errors); } this->errors.clear(); this->symbol_stack.clear(); From 9fece3fdf17ae642260ab8f8b85cb97a65bd14f7 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Tue, 25 Mar 2014 20:06:34 -0700 Subject: [PATCH 10/13] Space and time optimizations for parse_node_t. Reduced size from 48 bytes to 20 bytes. --- osx/config.h | 2 ++ parse_constants.h | 4 +-- parse_execution.cpp | 3 +- parse_tree.cpp | 69 ++++++++++++++++++++++++--------------------- parse_tree.h | 33 +++++++++++++--------- 5 files changed, 63 insertions(+), 48 deletions(-) diff --git a/osx/config.h b/osx/config.h index 4968a78b2..99f837427 100644 --- a/osx/config.h +++ b/osx/config.h @@ -218,7 +218,9 @@ #if __GNUC__ >= 3 #define __warn_unused __attribute__ ((warn_unused_result)) #define __sentinel __attribute__ ((sentinel)) +#define __packed __attribute__ ((packed)) #else #define __warn_unused #define __sentinel +#define __packed #endif diff --git a/parse_constants.h b/parse_constants.h index 8ff96407f..10818ffd0 100644 --- a/parse_constants.h +++ b/parse_constants.h @@ -70,7 +70,7 @@ enum parse_token_type_t LAST_TOKEN_OR_SYMBOL = parse_token_type_terminate, FIRST_PARSE_TOKEN_TYPE = parse_token_type_string -}; +} __packed; enum parse_keyword_t { @@ -93,7 +93,7 @@ enum parse_keyword_t parse_keyword_exec, LAST_KEYWORD = parse_keyword_builtin -}; +} __packed; /* Statement decorations. This matches the order of productions in decorated_statement */ enum parse_statement_decoration_t diff --git a/parse_execution.cpp b/parse_execution.cpp index 9267fb79b..484d8741e 100644 --- a/parse_execution.cpp +++ b/parse_execution.cpp @@ -72,7 +72,8 @@ node_offset_t parse_execution_context_t::get_offset(const parse_node_t &node) co const parse_node_t *addr = &node; const parse_node_t *base = &this->tree.at(0); assert(addr >= base); - node_offset_t offset = addr - base; + assert(addr - base < SOURCE_OFFSET_INVALID); + node_offset_t offset = static_cast(addr - base); assert(offset < this->tree.size()); assert(&tree.at(offset) == &node); return offset; diff --git a/parse_tree.cpp b/parse_tree.cpp index a1de4f966..7ea8481a9 100644 --- a/parse_tree.cpp +++ b/parse_tree.cpp @@ -13,6 +13,13 @@ static bool production_is_empty(const production_t *production) return (*production)[0] == token_type_invalid; } +void swap2(parse_node_tree_t &a, parse_node_tree_t &b) +{ + fprintf(stderr, "Swapping!\n"); + // This uses the base vector implementation + a.swap(b); +} + /** Returns a string description of this parse error */ wcstring parse_error_t::describe_with_prefix(const wcstring &src, const wcstring &prefix, bool skip_caret) const { @@ -404,7 +411,7 @@ static void dump_tree_recursive(const parse_node_tree_t &nodes, const wcstring & result->push_back(L'\n'); ++*line; - for (size_t child_idx = node.child_start; child_idx < node.child_start + node.child_count; child_idx++) + for (node_offset_t child_idx = node.child_start; child_idx < node.child_start + node.child_count; child_idx++) { dump_tree_recursive(nodes, src, child_idx, indent + 1, result, line, inout_first_node_not_dumped); } @@ -529,28 +536,31 @@ class parse_ll_t } // Get the parent index. But we can't get the parent parse node yet, since it may be made invalid by adding children - const size_t parent_node_idx = symbol_stack.back().node_idx; + const node_offset_t parent_node_idx = symbol_stack.back().node_idx; // Add the children. Confusingly, we want our nodes to be in forwards order (last token last, so dumps look nice), but the symbols should be reverse order (last token first, so it's lowest on the stack) - const size_t child_start = nodes.size(); - size_t child_count = 0; + const size_t child_start_big = nodes.size(); + assert(child_start_big < NODE_OFFSET_INVALID); + node_offset_t child_start = static_cast(child_start_big); + + // To avoid constructing multiple nodes, we push_back a single one that we modify + parse_node_t representative_child(token_type_invalid); + representative_child.parent = parent_node_idx; + + node_offset_t child_count = 0; for (size_t i=0; i < MAX_SYMBOLS_PER_PRODUCTION; i++) { production_element_t elem = (*production)[i]; - if (!production_element_is_valid(elem)) + if (! production_element_is_valid(elem)) { // All done, bail out break; } - else - { - // Generate the parse node. - parse_token_type_t child_type = production_element_type(elem); - parse_node_t child = parse_node_t(child_type); - child.parent = parent_node_idx; - nodes.push_back(child); - child_count++; - } + + // Append the parse node. + representative_child.type = production_element_type(elem); + nodes.push_back(representative_child); + child_count++; } // Update the parent @@ -566,7 +576,7 @@ class parse_ll_t // Replace the top of the stack with new stack elements corresponding to our new nodes. Note that these go in reverse order. symbol_stack.pop_back(); symbol_stack.reserve(symbol_stack.size() + child_count); - size_t idx = child_count; + node_offset_t idx = child_count; while (idx--) { production_element_t elem = (*production)[idx]; @@ -652,18 +662,17 @@ void parse_ll_t::dump_stack(void) const // Since children always appear after their parents, we can implement this very simply by walking backwards void parse_ll_t::determine_node_ranges(void) { - const size_t source_start_invalid = -1; size_t idx = nodes.size(); while (idx--) { - parse_node_t *parent = &nodes.at(idx); + parse_node_t *parent = &nodes[idx]; // Skip nodes that already have a source range. These are terminal nodes. - if (parent->source_start != source_start_invalid) + if (parent->source_start != SOURCE_OFFSET_INVALID) continue; // Ok, this node needs a source range. Get all of its children, and then set its range. - size_t min_start = source_start_invalid, max_end = 0; //note source_start_invalid is huge + source_offset_t min_start = SOURCE_OFFSET_INVALID, max_end = 0; //note SOURCE_OFFSET_INVALID is huge for (node_offset_t i=0; i < parent->child_count; i++) { const parse_node_t &child = nodes.at(parent->child_offset(i)); @@ -674,7 +683,7 @@ void parse_ll_t::determine_node_ranges(void) } } - if (min_start != source_start_invalid) + if (min_start != SOURCE_OFFSET_INVALID) { assert(max_end >= min_start); parent->source_start = min_start; @@ -831,7 +840,7 @@ void parse_ll_t::parse_error(const wchar_t *expected, parse_token_t token) void parse_ll_t::reset_symbols(enum parse_token_type_t goal) { /* Add a new goal node, and then reset our symbol list to point at it */ - node_offset_t where = nodes.size(); + node_offset_t where = static_cast(nodes.size()); nodes.push_back(parse_node_t(goal)); symbol_stack.clear(); @@ -1047,14 +1056,10 @@ static parse_keyword_t keyword_for_token(token_type tok, const wchar_t *tok_txt) } /* Placeholder invalid token */ -static const parse_token_t kInvalidToken = {token_type_invalid, -parse_keyword_none, false, false, static_cast(-1), - static_cast(-1)}; +static const parse_token_t kInvalidToken = {token_type_invalid, parse_keyword_none, false, false, SOURCE_OFFSET_INVALID, 0}; /* Terminal token */ -static const parse_token_t kTerminalToken = {parse_token_type_terminate, -parse_keyword_none, false, false, static_cast(-1), - static_cast(-1)}; +static const parse_token_t kTerminalToken = {parse_token_type_terminate, parse_keyword_none, false, false, SOURCE_OFFSET_INVALID, 0}; static inline bool is_help_argument(const wchar_t *txt) { @@ -1082,8 +1087,8 @@ static inline parse_token_t next_parse_token(tokenizer_t *tok) result.keyword = keyword_for_token(tok_type, tok_txt); result.has_dash_prefix = (tok_txt[0] == L'-'); result.is_help_argument = result.has_dash_prefix && is_help_argument(tok_txt); - result.source_start = (size_t)tok_start; - result.source_length = tok_extent; + result.source_start = (source_offset_t)tok_start; + result.source_length = (source_offset_t)tok_extent; tok_next(tok); return result; @@ -1195,7 +1200,7 @@ const parse_node_t *parse_node_tree_t::get_child(const parse_node_t &parent, nod const parse_node_t &parse_node_tree_t::find_child(const parse_node_t &parent, parse_token_type_t type) const { - for (size_t i=0; i < parent.child_count; i++) + for (node_offset_t i=0; i < parent.child_count; i++) { const parse_node_t *child = this->get_child(parent, i); if (child->type == type) @@ -1241,7 +1246,7 @@ static void find_nodes_recursive(const parse_node_tree_t &tree, const parse_node if (result->size() < max_count) { if (parent.type == type) result->push_back(&parent); - for (size_t i=0; i < parent.child_count; i++) + for (node_offset_t i=0; i < parent.child_count; i++) { const parse_node_t *child = tree.get_child(parent, i); assert(child != NULL); @@ -1478,7 +1483,7 @@ const parse_node_t *parse_node_tree_t::next_node_in_node_list(const parse_node_t const parse_node_t *next_cursor = NULL; /* Walk through the children */ - for (size_t i=0; i < list_cursor->child_count; i++) + for (node_offset_t i=0; i < list_cursor->child_count; i++) { const parse_node_t *child = this->get_child(*list_cursor, i); if (child->type == entry_type) diff --git a/parse_tree.h b/parse_tree.h index f6406be4f..2f0beb293 100644 --- a/parse_tree.h +++ b/parse_tree.h @@ -18,9 +18,15 @@ class parse_node_t; class parse_node_tree_t; -typedef size_t node_offset_t; + +typedef uint32_t node_offset_t; + #define NODE_OFFSET_INVALID (static_cast(-1)) +typedef uint32_t source_offset_t; + +#define SOURCE_OFFSET_INVALID (static_cast(-1)) + struct parse_error_t { /** Text of the error */ @@ -51,8 +57,8 @@ struct parse_token_t enum parse_keyword_t keyword; // Any keyword represented by this token bool has_dash_prefix; // Hackish: whether the source contains a dash prefix bool is_help_argument; // Hackish: whether the source looks like '-h' or '--help' - size_t source_start; - size_t source_length; + source_offset_t source_start; + source_offset_t source_length; wcstring describe() const; wcstring user_presentable_description() const; @@ -83,35 +89,36 @@ wcstring parse_dump_tree(const parse_node_tree_t &tree, const wcstring &src); wcstring token_type_description(parse_token_type_t type); wcstring keyword_description(parse_keyword_t type); -/** Class for nodes of a parse tree */ +/** Class for nodes of a parse tree. Since there's a lot of these, the size and order of the fields is important. */ class parse_node_t { public: - - /* Type of the node */ - enum parse_token_type_t type; - /* Start in the source code */ - size_t source_start; + source_offset_t source_start; /* Length of our range in the source code */ - size_t source_length; + source_offset_t source_length; /* Parent */ node_offset_t parent; /* Children */ node_offset_t child_start; + + /* Number of children */ uint8_t child_count; /* Which production was used */ uint8_t production_idx; + + /* Type of the node */ + enum parse_token_type_t type; /* Description */ wcstring describe(void) const; /* Constructor */ - explicit parse_node_t(parse_token_type_t ty) : type(ty), source_start(-1), source_length(0), parent(NODE_OFFSET_INVALID), child_start(0), child_count(0), production_idx(-1) + explicit parse_node_t(parse_token_type_t ty) : source_start(SOURCE_OFFSET_INVALID), source_length(0), parent(NODE_OFFSET_INVALID), child_start(0), child_count(0), production_idx(-1), type(ty) { } @@ -124,7 +131,7 @@ public: /* Indicate if this node has a range of source code associated with it */ bool has_source() const { - return source_start != (size_t)(-1); + return source_start != SOURCE_OFFSET_INVALID; } /* Gets source for the node, or the empty string if it has no source */ @@ -143,7 +150,6 @@ public: } }; - /* The parse tree itself */ class parse_node_tree_t : public std::vector { @@ -200,6 +206,7 @@ public: parse_node_list_t specific_statements_for_job(const parse_node_t &job) const; }; + /* The big entry point. Parse a string, attempting to produce a tree for the given goal type */ bool parse_tree_from_string(const wcstring &str, parse_tree_flags_t flags, parse_node_tree_t *output, parse_error_list_t *errors, parse_token_type_t goal = symbol_job_list); From 2db013a5fa379dae477e4bebc436ea387b674f93 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Tue, 25 Mar 2014 20:13:33 -0700 Subject: [PATCH 11/13] Hopeful fix for the build --- parse_constants.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parse_constants.h b/parse_constants.h index 10818ffd0..7b664383c 100644 --- a/parse_constants.h +++ b/parse_constants.h @@ -6,10 +6,11 @@ #ifndef fish_parse_constants_h #define fish_parse_constants_h +#include "config.h" + #define PARSE_ASSERT(a) assert(a) #define PARSER_DIE() do { fprintf(stderr, "Parser dying!\n"); exit_without_destructors(-1); } while (0) - enum parse_token_type_t { token_type_invalid, From d533c1b1c8288b738e4fbab2a0d58964d4c66d81 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Tue, 25 Mar 2014 20:23:58 -0700 Subject: [PATCH 12/13] Second attempt to fix the build. config.h.in changes need to go in configure.ac. --- configure.ac | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configure.ac b/configure.ac index f30187fbb..91f4f88cb 100644 --- a/configure.ac +++ b/configure.ac @@ -149,9 +149,11 @@ AC_CONFIG_HEADERS(config.h) AH_BOTTOM([#if __GNUC__ >= 3 #define __warn_unused __attribute__ ((warn_unused_result)) #define __sentinel __attribute__ ((sentinel)) +#define __packed __attribute__ ((packed)) #else #define __warn_unused #define __sentinel +#define __packed #endif]) From 7a7fb423b306d3de62ef62ff4c8cbded2cdd0f10 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Wed, 26 Mar 2014 10:00:32 -0700 Subject: [PATCH 13/13] Remove some unused function declarations --- highlight.cpp | 2 -- highlight.h | 5 ----- 2 files changed, 7 deletions(-) diff --git a/highlight.cpp b/highlight.cpp index 87985ae6b..d6db4c462 100644 --- a/highlight.cpp +++ b/highlight.cpp @@ -44,8 +44,6 @@ */ #define VAR_COUNT ( sizeof(highlight_var)/sizeof(wchar_t *) ) -static void highlight_universal_internal(const wcstring &buff, std::vector &color, size_t pos); - /** The environment variables used to specify the color of different tokens. This matches the order in highlight_spec_t */ static const wchar_t * const highlight_var[] = { diff --git a/highlight.h b/highlight.h index 6ef411f94..d411ae369 100644 --- a/highlight.h +++ b/highlight.h @@ -75,7 +75,6 @@ struct file_detection_context_t; \param error a list in which a description of each error will be inserted. May be 0, in whcich case no error descriptions will be generated. */ void highlight_shell(const wcstring &buffstr, std::vector &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars); -void highlight_shell_new_parser(const wcstring &buffstr, std::vector &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars); /** Perform syntax highlighting for the text in buff. Matching quotes and paranthesis are highlighted. The result is @@ -125,9 +124,5 @@ enum typedef unsigned int path_flags_t; bool is_potential_path(const wcstring &const_path, const wcstring_list_t &directories, path_flags_t flags, wcstring *out_path = NULL); -/* For testing */ -void highlight_shell_classic(const wcstring &buff, std::vector &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars); -void highlight_shell_new_parser(const wcstring &buff, std::vector &color, size_t pos, wcstring_list_t *error, const env_vars_snapshot_t &vars); - #endif