/* * This module tests unicode_reader and ustring in both the ICU and non-ICU case, * and tests these plus various mycu functions in the non-ICU case. * The Makefile builds this file two different ways, * potentially setting HAVE_ICU on the command-line. * * This file is part of Beastie * SPDX-FileCopyrightText: 2024 Norman Gray * SPDX-License-Identifier: BSD-2-Clause */ #if __GNUC__ && !defined(__clang__) // for fchdir and mkdtemp #define _XOPEN_SOURCE 700 #endif #include "config.h" #include #include #if HAVE_ALLOCA_H #include #endif // the following two are for the recursive directory delete #include #include #include "unicode.h" // HAVE_ICU is now set to 0 or 1 #include "uniprops.h" #include "ctype.h" #include "util.h" #include "c-unit.h" #if HAVE_ICU #include #endif // debugging: set to 1 to chatter more static int verbose = 0; static const char* progname; // given a unicode_reader, and a sequence of bytes, // check that the drained reader matches, // and then free the reader. #define ASSERT_READER_CONTENTS(rdr, nexpected, ...) \ do { \ unicode_reader* ur = (rdr); \ assert_not_null(ur); \ size_t nresults; \ codepoint_t* drained = drain_reader(ur, &nresults); \ assert_equal_int(nresults, nexpected); \ unicode_reader_free(ur); \ codepoint_t expected[] = { __VA_ARGS__ }; \ assert_equal_int32_array(drained, expected, nresults); \ } while (0) // Utility: delete directory and contents static void delete_directory_and_contents(const char* dirname) { if (verbose) fprintf(stderr, "%s: deleting directory %s...\n", progname, dirname); int currdir = open(".", O_RDONLY); if (chdir(dirname) != 0) { perror("can't chdir"); return; } DIR* td = opendir("."); struct dirent* d; while ((d = readdir(td)) != NULL) { //fprintf(stderr, " %s\n", d->d_name); if (d->d_name[0] != '.') { if (unlink(d->d_name) != 0) perror("failed to delete file"); } } closedir(td); if (fchdir(currdir) != 0) { perror("restoring currdir"); return; } close(currdir); if (rmdir(dirname) != 0) { perror("failed to delete directory"); } } #if !HAVE_ICU static codepoint_t decode(const unsigned char* str, int len) { const char* msg = NULL; codepoint_t rval = decode_utf8(str, len, NULL, (verbose ? &msg : NULL)); if (msg != NULL) { fprintf(stderr, "msg: %s\n", msg); free((void*)msg); } return rval; } #define T(str, len, exp) assert_equal_int(decode((unsigned char*)str, len), exp) static void test_good_decodings(void) { // extremes of the UTF-16 range T("\0", 1, 0); T("\xf4\x8f\xbf\xbf", 4, 0x10ffff); // RFC 3629, section 7 T("\x41", 1, 0x41); // A T("\xe2\x89\xa2", 3, 0x2262); // ≢ T("\xce\x91", 2, 0x391); // Α T("\xed\x95\x9c", 3, 0xd55c); /* 한 */ T("\xea\xb5\xad", 3, 0xad6d); /* 국 */ T("\xec\x96\xb4", 3, 0xc5b4); /* 어 */ T("\xe6\x97\xa5", 3, 0x65e5); /* 日 */ T("\xe6\x9c\xac", 3, 0x672c); /* 本 */ T("\xe8\xaa\x9e", 3, 0x8a9e); /* 語 */ T("\xef\xbb\xbf", 3, 0xfeff); /* BOM */ T("\xf0\xa3\x8e\xb4", 4, 0x233b4); // first and last encodings of a certain length (from Kuhn) T("", 1, 0); T("\x7f", 1, 0x7f); T("\xc2\x80", 2, 0x80); T("\xdf\xbf", 2, 0x7ff); T("\xe0\xa0\x80", 3, 0x800); T("\xef\xbf\xbf", 3, 0xffff); T("\xf0\x90\x80\x80", 4, 0x10000); // 0x1fffff would be encoded in 4 bytes, but is beyond the Unicode range // Kuhm lists the following as 'other boundary conditions' T("\xed\x9f\xbf", 3, 0xd7ff); // last codepoint before the UTF-16 surrogates T("\xee\x80\x80", 3, 0xe000); // first codepoint after the surrogates T("\xef\xbf\xbd", 3, 0xfffd); // the 'replacement character' T("\xf4\x8f\xbf\xbf", 4, 0x10ffff); // the last in-range codepoint T("\xf4\x90\x80\x80", 4, UNICODE_BAD_DECODE); // first out-of-range codepoint: should fail } static void test_bad_decodings(void) { // invalid encodings: these should return U+FFFD if displayed, // but decode_utf8 returns 0 if it detects any anomaly // There is a rather thorough list of invalid sequences, // and a quite general stress test, at // https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt // from where several of the tests below were taken T("\xff", 1, UNICODE_BAD_DECODE); // first byte not 11110xxx T("\x80\x20", 2, UNICODE_BAD_DECODE); // first byte is 10xxxxxx (continuation) T("\xb1\x20", 2, UNICODE_BAD_DECODE); // ...in the range of continuation bytes T("\xbf\x20", 2, UNICODE_BAD_DECODE); // ...last continuation byte T("\x80\xbf", 2, UNICODE_BAD_DECODE); // multiple continuation bytes T("\x80\xbf\x20", 2, UNICODE_BAD_DECODE); // multiple continuation bytes (followed by OK) T("\xc1\xc0", 2, UNICODE_BAD_DECODE); // second byte isn't 10xxxxxx T("\xe1\x81\xc0", 3, UNICODE_BAD_DECODE); // third byte isn't 10xxxxxx T("\xf1\x81\x81\xc0", 4, UNICODE_BAD_DECODE); // fourth byte isn't 10xxxxxx T("\xe2\x89", 2, UNICODE_BAD_DECODE); // short -- first byte indicates a 3-byte sequence T("\xe2\x20\x20", 3, UNICODE_BAD_DECODE); // byte 2 isn't a cont'n byte T("\xe2\x89\x20", 3, UNICODE_BAD_DECODE); // (final) byte 3 isn't a cont'n byte // decoding single UTF-16 surrogates should fail T("\xed\xa0\x80", 3, UNICODE_BAD_DECODE); // -> U+D800 T("\xed\xbf\xbf", 3, UNICODE_BAD_DECODE); // -> U+DFFF // decoding a pair of UTF-16 surrogates should also fail // (after decoding from UTF-8, the decoded codepoints would make // valid UTF-16, but that's not OK) T("\xed\xa0\x80\xed\xb0\x80", 6, UNICODE_BAD_DECODE); T("\xed\xaf\xbf\xed\xbf\xbf", 6, UNICODE_BAD_DECODE); // the following are overlong sequences for '/' T("\xc0\xaf", 2, UNICODE_BAD_DECODE); T("\xe0\x80\xaf", 3, UNICODE_BAD_DECODE); T("\xf0\x80\x80\xaf", 4, UNICODE_BAD_DECODE); T("\xf8\x80\x80\x80\xaf", 5, UNICODE_BAD_DECODE); T("\xfc\x80\x80\x80\x80\xaf", 6, UNICODE_BAD_DECODE); // U+110000 (the first codepoint out of range) is checked above // bad call to decode_utf8 with zero length -- shouldn't crash // (I may decide to make this an assertion failure instead) assert_equal_int(decode((unsigned char*)"a", 0), UNICODE_BAD_DECODE); // when we find an invalid start byte, confirm that we only consume one byte unsigned char nused; const char* errmsg; codepoint_t cp = decode_utf8((unsigned char*)"\xff\x81\x81\xc0", // first byte invalid 4, &nused, &errmsg); // fprintf(stderr, "decoding 0xff...: nused=%d, errmsg=%s\n", // nused, (errmsg ? errmsg : "")); assert_equal_int(cp, UNICODE_BAD_DECODE); assert_equal_int(nused, 1); } #undef T static void test_encodings(void) { #define T(cp, exp) do { \ int len; \ const unsigned char* b = encode_utf8(cp, &len); \ assert_equal_sn_fn(#cp, b, len, exp); \ } while (0) // see above for these sample codepoints T(0x41, "\x41"); T(0x391, "\xce\x91"); T(0x2262, "\xe2\x89\xa2"); T(0x233b4, "\xf0\xa3\x8e\xb4"); // for the following, see test_good_decodings above //T(0, ""); T(0x7f, "\x7f"); T(0x80, "\xc2\x80"); T(0x7ff, "\xdf\xbf"); T(0x800, "\xe0\xa0\x80"); T(0xffff, "\xef\xbf\xbf"); T(0x10000, "\xf0\x90\x80\x80"); // 0x1fffff would be encoded in 4 bytes, but is beyond the Unicode range T(0xd7ff, "\xed\x9f\xbf"); T(0xe000, "\xee\x80\x80"); T(0xfffd, "\xef\xbf\xbd"); T(0x10ffff, "\xf4\x8f\xbf\xbf"); // last in-range codepoint #undef T // refuse to encode certain codepoints #define T(cp) do { \ int len; \ assert_equal_ptr(encode_utf8(cp, &len), NULL); \ } while (0) T(0xd800); // high surrogate range T(0xdbff); // ...to here T(0xdc00); // low surrogate range T(0xdfff); // ...to here T(0x110000); // out of range #undef T } static void test_surrogates(void) { uint16_t s[2]; s[0] = 0xd834; s[1] = 0xdd1e; assert_equal_int(from_surrogate(s, NULL), 0x1d11e); // MUSICAL SYMBOL G CLEF s[0] = 0xd7ff; // last one before surrogates s[1] = 0xdd1e; assert_equal_int(from_surrogate(s, NULL), UNICODE_BAD_DECODE); s[0] = 0xdc00; // the first low surrogate s[1] = 0xdd1e; assert_equal_int(from_surrogate(s, NULL), UNICODE_BAD_DECODE); s[0] = 0xd834; s[1] = 0xdbff; // the last high surrogate assert_equal_int(from_surrogate(s, NULL), UNICODE_BAD_DECODE); s[0] = 0xd834; s[1] = 0xe000; // first one after the surrogates assert_equal_int(from_surrogate(s, NULL), UNICODE_BAD_DECODE); } #endif // !HAVE_ICU // XOR is true if precisely one of 'a' or 'b' is true. #define XOR(a,b) (!(a) != !(b)) static void test_uniprop_matches(void) { StringBuilder mismatches = make_stringbuilder(); stringbuilder_append_s(mismatches, "ctype mismatches: "); int nfails = 0; // confirm that we are testing the right functions // (the test is really that these link, rather than the result) #if HAVE_ICU assert_true(icu_is_icu_p()); #else assert_false(mycu_is_icu_p()); #endif // For each of the characters from 1 to 0x7f, confirm that the // UNIPROP_FUNC(X_p) predicate produces the same result as the // corresponding ctype one. for (char c=1; c<0x7f; c++) { int letter_bad = XOR(UNIPROP_FUNC(letter_p)(c), isalpha(c)); int upper_bad = XOR(UNIPROP_FUNC(uppercase_letter_p)(c), isupper(c)); int lower_bad = XOR(UNIPROP_FUNC(lowercase_letter_p)(c), islower(c)); int number_bad = XOR(UNIPROP_FUNC(number_p)(c), isdigit(c)); int space_bad = XOR(UNIPROP_FUNC(space_p)(c), isspace(c)); if (letter_bad + upper_bad + lower_bad + number_bad + space_bad) { nfails++; stringbuilder_printf(mismatches, "0x%x:%c%c%c%c%c;", c, (letter_bad ? 'a' : '-'), (upper_bad ? 'u' : '-'), (lower_bad ? 'l' : '-'), (number_bad ? 'n' : '-'), (space_bad ? 's' : '-')); } } char* fail_msg = NULL; if (nfails > 0) { stringbuilder_printf(mismatches, " (%d fails)", nfails); stringbuilder_terminate(mismatches); fail_msg = alloca(mismatches->len+1); memcpy(fail_msg, mismatches->buf, mismatches->len+1); } // tidy up stringbuilder_free(mismatches); if (fail_msg) { // the fail_msg is alloca-allocated, so is // implicitly freed when this function returns // (the string is printed within assert_fail, rather than // being passed anywhere) assert_fail(fail_msg); } else { assert_success("mycu"); } } static void test_uniprop_classes(void) { // confirm the classes of some sample characters assert_true(UNIPROP_FUNC(letter_p)('a')); assert_true(UNIPROP_FUNC(letter_p)(0xe9)); // é // 'अ'=U+0905 is the first uncomplicated letter encoded into 3 bytes assert_true(UNIPROP_FUNC(letter_p)(0x0905)); // 'ￜ' = U+ffdc appears to be the last letter in the BMP (still in 3 bytes) assert_true(UNIPROP_FUNC(letter_p)(0xffdc)); // LINEAR B SYLLABLE B008 A = U+10000 is the first character outside the BMP: // it is a letter #if HAVE_ICU assert_true(UNIPROP_FUNC(letter_p)(0x10000)); #else // ...but mycu doesn't (currently) support outside BMP assert_false(UNIPROP_FUNC(letter_p)(0x10000)); #endif assert_true(UNIPROP_FUNC(number_p)('0')); // U+ff19 is FULLWIDTH DIGIT NINE, and the last digit in the BMP assert_true(UNIPROP_FUNC(number_p)(0xff19)); // the first gap in the sequence of codepoints is just before U+037a // // 0377;GREEK SMALL LETTER PAMPHYLIAN DIGAMMA;Ll;0;L;;;;;N;;;0376;;0376 // 037A;GREEK YPOGEGRAMMENI;Lm;0;L; 0020 0345;;;;N;GREEK SPACING IOTA BELOW;;;; // ... // 037E;GREEK QUESTION MARK;Po;0;ON;003B;;;;N;;;;; assert_true(UNIPROP_FUNC(letter_p)(0x0377)); assert_false(UNIPROP_FUNC(number_p)(0x0377)); assert_false(UNIPROP_FUNC(letter_p)(0x0378)); // in the gap assert_false(UNIPROP_FUNC(number_p)(0x0378)); // in the gap assert_true(UNIPROP_FUNC(letter_p)(0x037a)); assert_false(UNIPROP_FUNC(number_p)(0x037a)); assert_false(UNIPROP_FUNC(letter_p)(0x037e)); assert_true(UNIPROP_FUNC(punctuation_p)(0x037e)); } static void test_uniprop_casefolding(void) { #define T(dir, c, expected) do { \ assert_equal_int(UNIPROP_FUNC(dir ## case_character)(c), expected); \ } while(0) // make sure to choose pairs which span a transition from letters to non-letters, // to check for off-by-ones T(upper, '`', 0x60); // '`' = 0x60, no uppercase T(upper, 'a', 0x41); T(upper, 'z', 0x5a); T(upper, 0x5b, 0x5b); // '[' = 0x5b, no uppercase T(upper, 0xdf, 0xdf); // LATIN SMALL LETTER SHARP S, no (simple) uppercase T(upper, 0xe0, 0xc0); // 'à' = 0xe0 T(lower, '@', 0x40); // '@" = 0x40, no lowercase T(lower, 'A', 0x61); T(lower, 'Z', 0x7a); T(lower, '{', 0x7b); // '{' = 0x7b, no lowercase T(lower, 0xbf, 0xbf); // INVERTED QUESTION MARK, no lowercase T(lower, 0xc0, 0xe0); // 'À' = 0xc0 // most titlecases are the same as lowercase T(title, 'a', 0x41); T(title, 'A', 0x41); // titlecase same // but not all... // // 01C4;LATIN CAPITAL LETTER DZ WITH CARON;Lu;0;L; 0044 017D;;;;N;LATIN CAPITAL LETTER D Z HACEK;;;01C6;01C5 // 01C5;LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON;Lt;0;L; 0044 017E;;;;N;LATIN LETTER CAPITAL D SMALL Z HACEK;;01C4;01C6;01C5 // 01C6;LATIN SMALL LETTER DZ WITH CARON;Ll;0;L; 0064 017E;;;;N;LATIN SMALL LETTER D Z HACEK;;01C4;;01C5 // // That is: // | upper | lower | title // -----|-------|-------|------ // 01c4 | - | 01c6 | 01c5 // 01c5 | 01c4 | 01c6 | 01c5 // 01c6 | 01c4 | - | 01c5 T(upper, 0x1c4, 0x1c4); // uppercase same T(lower, 0x1c4, 0x1c6); T(title, 0x1c4, 0x1c5); T(upper, 0x1c5, 0x1c4); T(lower, 0x1c5, 0x1c6); T(title, 0x1c5, 0x1c5); // titlecase same T(upper, 0x1c6, 0x1c4); T(lower, 0x1c6, 0x1c6); // lowercase same T(title, 0x1c6, 0x1c5); // the first gap in the sequence of codepoints is just before U+037a // // 0377;GREEK SMALL LETTER PAMPHYLIAN DIGAMMA;Ll;0;L;;;;;N;;;0376;;0376 // 037A;GREEK YPOGEGRAMMENI;Lm;0;L; 0020 0345;;;;N;GREEK SPACING IOTA BELOW;;;; // 037B;GREEK SMALL REVERSED LUNATE SIGMA SYMBOL;Ll;0;L;;;;;N;;;03FD;;03FD T(upper, 0x37a, 0x37a); // uppercase same T(upper, 0x37b, 0x03fd); T(lower, 0x37b, 0x37b); // lowecase same T(title, 0x37b, 0x03fd); } // debugging function: returns a pointer to a static array static codepoint_t* drain_reader(unicode_reader* ur, size_t* nresults) { #define BUFLEN 64 static codepoint_t results[BUFLEN]; size_t nr = 0; codepoint_t cp = unicode_reader_next_cp(ur, NULL); while (cp != UNICODE_EOF) { if (cp == UNICODE_BAD_DECODE) { results[nr++] = UNICODE_REPLACEMENT_CHARACTER; } else { results[nr++] = cp; } if (nr == BUFLEN) { // eh? -- just give up fprintf(stderr, "test-unicode: drain_reader buf too small!\n"); exit(1); } cp = unicode_reader_next_cp(ur, NULL); } if (nresults != NULL) *nresults = nr; return results; #undef BUFLEN } // '¢'=U+A2 or '£'=U+A3 are the lowest letter-like codepoints // (U+A0 is NBSP, which I'm uncertain about permitting, and U+A1 is // upside-down-exclamation); character 'ߧ'=U+07e7 is 'NKO LETTER NYA // WOLOSO' and is nearly the last 2-byte-encoded letter, 'अ'=U+0905 // is the first uncomplicated letter encoded into 3 bytes, and // '𐀀𐀀'=U+10000 is from Linear B, and is the first encoded in 4 bytes static void test_file_reads(void) { char template[] = "/tmp/test-unicode-XXXXXX"; char* tempdir = mkdtemp(template); if (verbose) fprintf(stderr, "%s: tempdir=%s\n", progname, tempdir); // our temporary/test file names below are all shorter than 32 chars long static const size_t fnbuf_len = sizeof(template) + 32; char fnbuf[fnbuf_len]; // a simple file, with a mix of character sizes snprintf(fnbuf, fnbuf_len, "%s/simple.txt", tempdir); FILE* tfile = fopen(fnbuf, "w"); fprintf(tfile, "aé¢ߧअ𐀀\nx\n"); fclose(tfile); // codepoint_t* results; // size_t nresults = 0; ASSERT_READER_CONTENTS(make_unicode_reader_file(fnbuf, 64, NULL), 9, 0x61, 0xe9, 0xa2, 0x07e7, 0x0905, 0x10000, '\n', //0x0a, 'x', '\n'); // a longer file, to exercise multiple buffer reads snprintf(fnbuf, fnbuf_len, "%s/long.txt", tempdir); tfile = fopen(fnbuf, "w"); // write 8 bytes fprintf(tfile, "éééé"); // 9 bytes, so the last é spans a buffer boundary fprintf(tfile, "aéééé"); // 3 x 3 bytes, so the last character is also split fprintf(tfile, "अअअ"); // ...adding to 3 x 8 + 2 bytes fclose(tfile); ASSERT_READER_CONTENTS(make_unicode_reader_file(fnbuf, 8, NULL), 12, 0xe9, 0xe9, 0xe9, 0xe9, 0x61, 0xe9, 0xe9, 0xe9, 0xe9, 0x905, 0x905, 0x905 ); // read the same string as above (simple.txt), // but this time from a string rather than a file ASSERT_READER_CONTENTS(make_unicode_reader_string((unsigned char*)"aé¢ߧअ𐀀\n", NULL), 7, 0x61, 0xe9, 0xa2, 0x07e7, 0x0905, 0x10000, 0x0a); // reading a bytestring with invalid content ASSERT_READER_CONTENTS(make_unicode_reader_string((unsigned char*)"\x41\xff\x42", NULL), 3, 0x41, UNICODE_REPLACEMENT_CHARACTER, 0x42); delete_directory_and_contents(tempdir); // pushbacks and EOF... unicode_reader* ur = make_unicode_reader_string((unsigned char*)"ab", NULL); assert_equal_int(unicode_reader_next_cp(ur, NULL), 'a'); push_codepoint(ur, 'a'); assert_equal_int(unicode_reader_next_cp(ur, NULL), 'a'); assert_equal_int(unicode_reader_next_cp(ur, NULL), 'b'); push_codepoint(ur, 'c'); assert_equal_int(unicode_reader_next_cp(ur, NULL), 'c'); assert_equal_int(unicode_reader_next_cp(ur, NULL), UNICODE_EOF); assert_equal_int(unicode_reader_next_cp(ur, NULL), UNICODE_EOF); push_codepoint(ur, UNICODE_EOF); assert_equal_int(unicode_reader_next_cp(ur, NULL), UNICODE_EOF); } // testing the ICU functions here confirms that the tests are correct static void test_whitespace(void) { #define YES(cp) assert_true(UNIPROP_FUNC(space_p)(cp)) #define NO(cp) assert_false(UNIPROP_FUNC(space_p)(cp)) NO(0x08); YES(0x09); // HT YES(0x0a); // LF YES(0x0b); // VT YES(0x0c); // NP YES(0x0d); // CR NO(0x0e); YES(0x85); // NEL YES(0xa0); // NBSP YES(0x1680); // OGHAM SPACE YES(0x2000); // EN QUAD SPACE YES(0x200a); // HAIR SPACE NO(0x200b); // ...after range YES(0x2028); // LS YES(0x2029); // PS YES(0x202f); // NARROW NO-BREAK SPACE NO(0x2030); // ...for example YES(0x3000); // IDEOGRAPHIC SPACE #undef YES #undef NO // See the definition of isWhitespace at // https://unicode-org.github.io/icu-docs/apidoc/dev/icu4c/uchar_8h.html #define YES(cp) assert_true(UNIPROP_FUNC(whitespace_p)(cp)) #define NO(cp) assert_false(UNIPROP_FUNC(whitespace_p)(cp)) // From that documentation: // A character is considered to be a Java whitespace character // if and only if it satisfies one of the following criteria: // // It is a Unicode Separator character (categories "Z" = "Zs" or "Zl" or "Zp"), // but is not also a non-breaking space // (U+00A0 NBSP or U+2007 Figure Space or U+202F Narrow NBSP). // Referring to PropList.txt (Zs except where noted) NO('a'); // random non-space character YES(0x0020); // SPACE NO(0x00A0); // NO-BREAK SPACE YES(0x1680); // OGHAM SPACE MARK YES(0x2000); // EN QUAD NO(0x2007); // Figure space YES(0x200A); // ..HAIR SPACE YES(0x2028); // LINE SEPARATOR (Zl) YES(0x2029); // PARAGRAPH SEPARATOR (Zp) NO(0x202F); // NARROW NO-BREAK SPACE YES(0x205F); // MEDIUM MATHEMATICAL SPACE YES(0x3000); // IDEOGRAPHIC SPACE // or YES(0x0009); // HORIZONTAL TABULATION. YES(0x000A); // LINE FEED. YES(0x000B); // VERTICAL TABULATION. YES(0x000C); // FORM FEED. YES(0x000D); // CARRIAGE RETURN. YES(0x001C); // FILE SEPARATOR. YES(0x001D); // GROUP SEPARATOR. YES(0x001E); // RECORD SEPARATOR. YES(0x001F); // UNIT SEPARATOR. #undef YES #undef NO #define YES(cp) assert_true(UNIPROP_FUNC(nonbreakingspace_p)(cp)) #define NO(cp) assert_false(UNIPROP_FUNC(nonbreakingspace_p)(cp)) NO(' '); YES(0x00a0); YES(0x2007); YES(0x202f); #undef YES #undef NO } static void test_uniprop_properties(void) { #define YES(cp) assert_true(UNIPROP_FUNC(alphabetic_p)(cp)) #define NO(cp) assert_false(UNIPROP_FUNC(alphabetic_p)(cp)) // in the ASCII range -- fast path NO('@'); YES('A'); YES('Z'); NO('['); NO('`'); YES('a'); YES('z'); NO('{'); // lowest non-ASCII alphabetic is 0xaa NO(0xa9); YES(0xaa); NO(0xab); // the (devanagari) vowel signs are oddly laid out, with a mixture // of single-codepoint ranges right alongside contiguous ranges: // // 0949..094C ; Alphabetic # Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU // 094E..094F ; Alphabetic # Mc [2] DEVANAGARI VOWEL SIGN PRISHTHAMATRA E..DEVANAGARI VOWEL SIGN AW // 0950 ; Alphabetic # Lo DEVANAGARI OM // 0955..0957 ; Alphabetic # Mn [3] DEVANAGARI VOWEL SIGN CANDRA LONG E..DEVANAGARI VOWEL SIGN UUE // // check these edge cases NO(0x94d); YES(0x94f); // end of range YES(0x950); // single-codepoint 'range' NO(0x951); // end of the range YES(0xffdc); NO(0xffdd); NO(0xffff); // beyond BMP #if HAVE_ICU YES(0x10000); #else NO(0x10000); #endif // Now do the same for the 'word character' test. // These are mostly redundant with the above. #define WYES(cp) assert_true(UNIPROP_FUNC(wordcharacter_p)(cp)) #define WNO(cp) assert_false(UNIPROP_FUNC(wordcharacter_p)(cp)) NO('@'); WNO('@'); YES('A'); WYES('A'); NO(0x094d); // DEVANAGARI SIGN VIRAMA, a diacritic WYES(0x094d); NO(0x200c); // ZERO WIDTH NON-JOINER (ZWNJ) WYES(0x200c); NO(0x200d); // ZERO WIDTH JOINER (ZWJ) WYES(0x200d); NO(0xb7); // MIDDLE DOT WYES(0xb7); NO(0xffe3); // FULLWIDTH MACRON WYES(0xffe3); WNO(0xffe4); // +1 #undef WYES #undef WNO #undef YES #undef NO #if !HAVE_ICU // declare unexposed function (which is called within mycu_wordcharacter_p) int mycu_extender_p(const uint32_t cp); // the following is an edge-case: // the argument 0xb6 is lower than map[0].start // (and outside than the fast-path cases in other predicates) assert_false(mycu_extender_p(0xb6)); assert_true(mycu_extender_p(0xb7)); // same, at the other end of the range // (we've really checked this case above, but include this for symmetry, // and in case this is more edgy than I expect) assert_true(mycu_extender_p(0xff70)); assert_false(mycu_extender_p(0xff71)); #endif } // checks of cache_store: // at various points below, the cache should have become NULL: // check that, and make it non-null again. // Thus, after calling DUMMY_CACHE after each make_ustring, // we should be able to assert CACHE_IS_NULL // after any operation which changes the ustring const char* dummy_cache = "boo!"; #define DUMMY_CACHE(us) do { \ assert_not_null(us); \ ustring_cache_store_set((us), (const void*)dummy_cache); \ } while (0) #define CACHE_IS_NULL(us) do { \ assert_not_null(us); \ assert_null(ustring_cache_store_get(us)); \ DUMMY_CACHE(us); \ } while (0) // There are several tests involving ustring->s_ and ustring->idx_ here. // These are indeed internal fields of the ustring structure, so I // should do something different here, but this whole module is // somewhat internal, so this is a venial sin. static void test_ustring(void) { // working ustrings (ustring_reset before reuse) ustring_t us = make_ustring(NULL); DUMMY_CACHE(us); assert_equal_s((char*)ustring_cache_store_get(us), dummy_cache); // not all of the following are assigned codepoints // one byte in UTF-8 ustring_append_cp(us, 0x20, NULL); ustring_append_cp(us, 0x7f, NULL); CACHE_IS_NULL(us); // two bytes in UTF-8 ustring_append_cp(us, 0x80, NULL); // ustring_append_cp(us, 0xe9); // é: two bytes in UTF-8 // ustring_append_cp(us, 0x0621); // ARABIC LETTER HAMZA ustring_append_cp(us, 0x07ff, NULL); // three bytes in UTF-8 ustring_append_cp(us, 0x0800, NULL); // ustring_append_cp(us, 0x0800); // SAMARITAN LETTER ALAF // ustring_append_cp(us, 0xd7fb); // HANGUL JONGSEONG PHIEUPH-THIEUTH, nearly at the surrogates ustring_append_cp(us, 0xd7ff, NULL); // last before surrogates ustring_append_cp(us, 0xe000, NULL); // first after surrogates (private use area) ustring_append_cp(us, 0xfffd, NULL); // last character in BMP // four bytes in UTF-8 ustring_append_cp(us, 0x10000, NULL); // U+10fffd is the last character codepoint (private-use area) ustring_append_cp(us, 0x10fffd, NULL); assert_equal_uint(ustring_length(us), 10); assert_equal_uint(us->idx_, 12); uint16_t expected_utf16[] = { 0x20, 0x7f, 0x0080, 0x07ff, 0x0800, 0xd7ff, 0xe000, 0xfffd, 0xd800, 0xdc00, // U+10000 0xdbff, 0xdffd, // U+10FFFD }; const size_t n_expected_utf16 = sizeof(expected_utf16)/sizeof(expected_utf16[0]); uint8_t expected_utf8[] = { 0x20, 0x7f, 0xc2, 0x80, // U+0080 0xdf, 0xbf, // U+07ff 0xe0, 0xa0, 0x80, // U+0800 0xed, 0x9f, 0xbf, // U+d7ff 0xee, 0x80, 0x80, // U+e000 0xef, 0xbf, 0xbd, // U+fffd 0xf0, 0x90, 0x80, 0x80, // U+10000 0xf4, 0x8f, 0xbf, 0xbd, // U+10fffd }; const size_t n_expected_utf8 = sizeof(expected_utf8)/sizeof(expected_utf8[0]); assert_equal_uint16_array(us->s_, expected_utf16, n_expected_utf16); const uint8_t* b8 = ustring_to_utf8(us, NULL); assert_equal_uint8_array(b8, expected_utf8, n_expected_utf8); free((void*)b8); ustring_reset(us); CACHE_IS_NULL(us); // appending the following codepoints should fail assert_null(ustring_append_cp(us, 0x110000, NULL)); // too big assert_null(ustring_append_cp(us, 0xd800, NULL)); // surrogate assert_null(ustring_append_cp(us, 0xdfff, NULL)); // surrogate assert_null(ustring_append_cp(us, 0x1fffe, NULL)); // 2nd last in plane assert_null(ustring_append_cp(us, 0xfdd0, NULL)); // process-internal assert_equal_uint(ustring_length(us), 0); assert_equal_uint(us->idx_, 0); ustring_reset(us); ustring_append_cp(us, 'a', NULL); ustring_append_cp(us, 0xe9, NULL); assert_equal_uint(ustring_ref(us, 0), 'a'); assert_equal_uint(ustring_ref(us, 1), 0xe9); assert_equal_uint(ustring_ref(us, 2), 0); // end of string assert_equal_uint(ustring_ref(us, -1), 0); // before beginning of string // test_cp requires two code-units, // and has different bits set in the two surrogates const codepoint_t test_cp = 0x10000 | (0x1 << 12) | 1; ustring_append_cp(us, test_cp, NULL); // requires two code-units assert_equal_uint(ustring_ref(us, 2), test_cp); // while this is the last character in the string ustring_append_cp(us, 'a', NULL); // just one code-unit, but on the far side of the above assert_equal_uint(ustring_ref(us, 2), test_cp); // ...still assert_equal_uint(ustring_ref(us, 3), 'a'); assert_equal_uint(ustring_ref(us, 4), 0); // end of string ustring_reset(us); ustring_append_cp(us, test_cp, NULL); // first character in string ustring_append_cp(us, 'a', NULL); ustring_append_cp(us, 'b', NULL); assert_equal_uint(ustring_ref(us, 0), test_cp); assert_equal_uint(ustring_ref(us, 1), 'a'); // on the far side of char-0 assert_equal_uint(ustring_ref(us, 2), 'b'); // on the far side of char-0 ustring_t us2 = make_ustring(NULL); DUMMY_CACHE(us2); ustring_append_utf8(us2, (uint8_t*)"héءx", NULL); CACHE_IS_NULL(us2); assert_equal_uint(ustring_length(us2), 4); assert_equal_uint(us2->idx_, 4); assert_equal_uint(ustring_ref(us2, 0), 'h'); assert_equal_uint(ustring_ref(us2, 1), 0xe9); assert_equal_uint(ustring_ref(us2, 2), 0x0621); assert_equal_uint(ustring_ref(us2, 3), 'x'); ustring_append_ustring(us, us2, NULL); assert_equal_uint(ustring_length(us), 7); assert_equal_uint(us->idx_, 8); //for (int i=0; iidx; i++) printf(" %d: %x\n", i, us->s[i]); //printf("chars resulting...\n"); //for (int i=0; ilen; i++) printf(" %d: %x\n", i, ustring_ref(us, i)); assert_equal_uint(ustring_ref(us, 6), 'x'); // testing ustring_append_ustring ustring_reset(us); ustring_reset(us2); ustring_append_utf8(us2, (byte_t*)"ab", NULL); ustring_append_ustring(us, us2, NULL); // both now "ab" assert_true(ustring_equal(us, us2, 0)); CACHE_IS_NULL(us); CACHE_IS_NULL(us2); ustring_reset(us2); assert_equal_int(ustring_length(us2), 0); ustring_append_ustring(us, us2, NULL); // us is still "ab" assert_equal_int(ustring_length(us), 2); // original length // ustring_substring ustring_reset(us); ustring_append_utf8(us, (byte_t*)"01234567", NULL); CACHE_IS_NULL(us); // and sets cache to non-null dummy value const char* errmsg = NULL; // unsuccessful substring operations ustring_t ss = ustring_substring(us, -1, -1, &errmsg); errmsg = NULL; // reset assert_null(ss); ss = ustring_substring(us, 4, 2, &errmsg); errmsg = NULL; // reset assert_null(ss); // a successful substring operation ss = ustring_substring(us, 0, -1, &errmsg); assert_not_null(ss); CACHE_IS_NULL(ss); // but the source should be unchanged assert_not_null(ustring_cache_store_get(us)); assert_equal_s((const char*)ustring_cache_store_get(us), "boo!"); // still ustring_reset(us2); ustring_append_utf8(us2, (byte_t*)"01234567", NULL); assert_true(ustring_equal(ss, us2, 0)); ustring_free(ss); ss = ustring_substring(us, 0, 4, &errmsg); if (errmsg != NULL) fprintf(stderr, "ustring_substring warning: %s\n", errmsg); assert_not_null(ss); ustring_reset(us2); ustring_append_utf8(us2, (byte_t*)"0123", NULL); assert_true(ustring_equal(ss, us2, 0)); ustring_free(ss); ss = ustring_substring(us, 0, 0, &errmsg); if (errmsg != NULL) fprintf(stderr, "ustring_substring warning: %s\n", errmsg); assert_not_null(ss); assert_equal_int(ustring_length(ss), 0); ustring_free(ss); ss = ustring_substring(us, 4, -1, &errmsg); if (errmsg != NULL) fprintf(stderr, "ustring_substring warning: %s\n", errmsg); assert_not_null(ss); ustring_reset(us2); ustring_append_utf8(us2, (byte_t*)"4567", NULL); assert_true(ustring_equal(ss, us2, 0)); ustring_free(ss); ss = ustring_substring(us, 4, 6, &errmsg); if (errmsg != NULL) fprintf(stderr, "ustring_substring warning: %s\n", errmsg); assert_not_null(ss); ustring_reset(us2); ustring_append_utf8(us2, (byte_t*)"45", NULL); assert_true(ustring_equal(ss, us2, 0)); ustring_free(ss); ustring_reset(us); ustring_append_cp(us, 'a', NULL); ustring_append_cp(us, 0xe9, NULL); // é ustring_append_cp(us, 0x10000, NULL); // LINEAR B SYLLABLE B008 A (outside BMP) ustring_append_cp(us, 'a', NULL); ss = ustring_substring(us, 0, -1, NULL); assert_not_null(ss); ustring_reset(us2); ustring_append_utf8(us2, (byte_t*)"aé𐀀a", NULL); assert_true(ustring_equal(ss, us2, 0)); ustring_free(ss); ss = ustring_substring(us, 2, 3, NULL); assert_not_null(ss); ustring_reset(us2); ustring_append_utf8(us2, (byte_t*)"𐀀", NULL); assert_true(ustring_equal(ss, us2, 0)); ustring_free(ss); // the above doesn't test end being longer than the string // (because I'm not sure whether that should be an actual error). // ustring_append_utf8 ustring_reset(us); ustring_append_utf8(us, (uint8_t*)"h\xf0\x90\x80\x80x", NULL); // U+10000 assert_equal_uint(ustring_length(us), 3); // 3 characters assert_equal_uint(us->idx_, 4); // ...but 4 code units // The following tests overlap with those in test_bad_decodings to // some extent, but by testing with ustrings, we are one layer // away from decode_utf8. The tests in test-unicode.scm are // similar, but at least one call further out from decode_utf8. ustring_reset(us); DUMMY_CACHE(us); ustring_append_utf8(us, (unsigned char*)"a\xff""b", NULL); ASSERT_READER_CONTENTS(make_unicode_reader_ustring(us, NULL), 3, 'a', UNICODE_REPLACEMENT_CHARACTER, 'b' ); CACHE_IS_NULL(us); ustring_reset(us); // This test has to be done slightly differently, since the ICU // and non-ICU versions handle this in different (and I think both // legitimate) ways. My non-ICU version returns a single 0xfffd // for the overlong 'P', but ICU returns three of them. So use // the ustring_equal function with the appropriate flag. // // As it happens, the ICU and non-ICU versions behave identically // in the other replacement-character situations, so we don't need // this longer process to check their equality. assert_not_null(ustring_append_utf8(us, // 'é' and overlong 'P' (unsigned char*)"\xc3\xa9\xe0\x80\xaf", NULL)); ustring_reset(us2); ustring_append_utf8(us2, (const unsigned char*)"é���", NULL); // ustring_append_cp(us2, 0xe9, NULL); // ustring_append_cp(us2, UNICODE_REPLACEMENT_CHARACTER, NULL); assert_true(ustring_equal(us, us2, USTRING_EQUAL_COLLAPSE_REPLACEMENTS)); ustring_reset(us); ustring_append_utf8(us, (unsigned char*)"a" "\xc3\xa9" //é "\xff" // invalid "\xc3\xbc" // ü "b", NULL); ASSERT_READER_CONTENTS(make_unicode_reader_ustring(us, NULL), 5, 0x61, 0xe9, 0xfffd, 0xfc, 0x62); } static void test_unicode_collation(void) { // generic/simple/POSIX tests ustring_t us_a = make_ustring(NULL); ustring_append_utf8(us_a, (uint8_t*)"a", NULL); ustring_t us_aa = make_ustring(NULL); ustring_append_utf8(us_aa, (uint8_t*)"aa", NULL); ustring_t us_b = make_ustring(NULL); ustring_append_utf8(us_b, (uint8_t*)"b", NULL); ustring_t us_null = make_ustring(NULL); DUMMY_CACHE(us_a); // shouldn't be affected at all by the tests below assert_true (ustring_lt(us_a, us_b)); assert_false(ustring_lt(us_b, us_a)); assert_true (ustring_lt(us_a, us_aa)); assert_false(ustring_lt(us_aa, us_a)); assert_false(ustring_lt(us_a, us_a)); assert_false(ustring_lt(us_null, us_null)); assert_true (ustring_lt(us_null, us_a)); assert_false(ustring_lt(us_a, us_null)); #if HAVE_ICU // language-dependent sorting: this is the example // illustrated in https://www.unicode.org/reports/tr10/ ustring_t us_oog = make_ustring(NULL); ustring_append_utf8(us_oog, (byte_t*)"öog", NULL); ustring_t us_zog = make_ustring(NULL); ustring_append_utf8(us_zog, (byte_t*)"zog", NULL); unicode_set_locale("DE", NULL); assert_true(ustring_lt(us_oog, us_zog)); unicode_set_locale("sv", NULL); assert_false(ustring_lt(us_oog, us_zog)); #endif assert_not_null(ustring_cache_store_get(us_a)); assert_equal_s((const char*)ustring_cache_store_get(us_a), "boo!"); } s7_scheme* S7; void Usage(void) { fprintf(stderr, "Usage: %s [-v]\n", progname); exit(1); } int main(int argc, char** argv) { progname = argv[0]; for (argc--, argv++; argc>0; argc--, argv++) { if (**argv == '-') { switch (*++*argv) { case 'v': verbose = 1; break; default: Usage(); } } else { Usage(); } } #if HAVE_ICU printf(" (with ICU " U_ICU_VERSION ")\n"); #else printf(" (without ICU)\n"); #endif { char* init_errmsg = NULL; unsetenv("BEASTIE_LOCALE"); // ...for tests if (initialise_unicode_module(&init_errmsg) != 0) { fprintf(stderr, "Unable to initialise the unicode module! (%s)\n", init_errmsg); exit(1); } } int rval; #if !HAVE_ICU run_test_suite("decoding good sequences", test_good_decodings); run_test_suite("decoding bad sequences", test_bad_decodings); run_test_suite("encoding", test_encodings); run_test_suite("UTF-16 surrogates", test_surrogates); #endif run_test_suite("uniprops", test_uniprop_matches); run_test_suite("uniprops classes", test_uniprop_classes); run_test_suite("uniprops casefolding", test_uniprop_casefolding); run_test_suite("uniprops properties", test_uniprop_properties); run_test_suite("uniprops whitespace", test_whitespace); run_test_suite("unicode_reader", test_file_reads); run_test_suite("ustring", test_ustring); run_test_suite("unicode collation", test_unicode_collation); exit(report_status()); }