// Encoding and decoding UTF-8; defining the ustring type.
//
// See
//
//   * Unicode standard, Sects. 2.5 and 3.9:
//     https://www.unicode.org/versions/Unicode15.1.0/ch02.pdf#G13708
//     https://www.unicode.org/versions/Unicode15.1.0/ch03.pdf#G7404
//   * RFC 2781: UTF-16, an encoding of ISO 10646 -- https://www.ietf.org/rfc/rfc2781.txt
//   * RFC 3629: UTF-8, a transformation format of ISO 10646 -- https://www.ietf.org/rfc/rfc3629.txt
//   * Unicode FAQ: https://www.unicode.org/faq/utf_bom.html
//   * Markus Kuhn's UTF-8 decoder stess tests: https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
//
// I deliberately don't include s7.h in this module,
// since I want to keep this generic (for general tidiness/orthogonality reasons,
// rather than because I necessarily want to move these functions elsewhere).
// This means that some out-of-memory errors, which should really result in
// a return_beastie_error or error_exit (both of which rely on the global S7 variable),
// simply return NULL with an error message if possible.
//
// The ustring_t type is intended as a way of storing Unicode strings
// internally to this program, in a way which is reasonably efficient
// to store and manipulate, compared with storing everying as UTF-8
// internally (which is implicitly mostly done by s7), or as lists of
// s7 integers (which works, but feels terribly clumsy, and the weak
// typing of which requires one to remember whether a list is 'just' a
// list, or in fact a list of codepoints).  It means that strings are
// stored as 16 bits per codepoint (if everything's in the BMP) rather
// than 64 bits (if we were using lists of integers, since s7_int is int64_t).
//
// The encoding of codepoints
// here is almost identical to that of UTF-16, but isn't _quite_ the
// same thing, since it does so using a sequence of uint16_t integers,
// rather than a sequence of big- or little-endian bytes.
//
// The code here includes versions which do and don't use ICU.
// The ICU API docs are at
// https://unicode-org.github.io/icu-docs/apidoc/released/icu4c/index.html
//
// This file is part of Beastie <https://purl.org/nxg/dist/beastie>
// SPDX-FileCopyrightText: 2024 Norman Gray <https://nxg.me.uk>
// SPDX-License-Identifier: BSD-2-Clause

#if __GNUC__
// for stpncpy
#define _XOPEN_SOURCE 700
#endif

#include <stdio.h>
#include <stdarg.h>
#include <string.h>
#include <assert.h>
#include <errno.h>

#include "unicode.h"

#if HAVE_ICU
#include <unicode/ustdio.h>
#include <unicode/ustring.h>
#include <unicode/utf.h>
#include <unicode/ucol.h>
#include <unicode/uloc.h>
#endif

static int verbosity_ = 1;
int unicode_verbosity(int increment)
{
    if (increment > 0) {
        ++verbosity_;
    } else if (increment < 0 && verbosity_ > 0) {
        --verbosity_;
    }
    return verbosity_;
}

// Return a static string containing the
// UCD version this build is based upon.
const char* unicode_version()
{
#if HAVE_ICU

#define VLEN 16
    static char version_string[VLEN] = "";
    if (version_string[0] == '\0') {
        UVersionInfo unicode_version;
        u_getUnicodeVersion(unicode_version);
        snprintf(version_string, VLEN, "%d.%d.%d",
                 unicode_version[0],
                 unicode_version[1],
                 unicode_version[2]);
    }
    return version_string;
#undef VLEN

#else

#include "misc/unicode/ucd/ucd_version.h"
    return ucd_version;

#endif
}

// No, the following isn't thread-safe, but if we allocate memory here
// we have to free it later, and this error message is
// possibly/probably going to end up being passed to
// return_beastie_error, which inevitably returns before freeing
// anything.  A way of getting round this would be to create and
// return a scheme object, but that would require pulling s7.h in here
// in a way that feels inelegant just now.  Perhaps come back to this later.
//
// Since this error message is probably terminal, we seem likely to be
// exiting soon anyway, so that this in-principle thread-unsafety
// doesn't much matter.
#define ERRMSG_BUF_LEN 1024
static char errmsg_buf[ERRMSG_BUF_LEN];
static const char* create_errmsg(const char* fmt, ...)
{
    va_list ap;
    va_start(ap, fmt);
    vsnprintf(errmsg_buf, ERRMSG_BUF_LEN, fmt, ap);
    va_end(ap);
    return errmsg_buf;
}
#if 0
// a dynamic version, but not to be used...
static const char* create_errmsg(const char* fmt, ...)
{
    char* msg;
    va_list ap;
    va_start(ap, fmt);
    int status = vasprintf(&msg, fmt, ap);
    if (status < 0) {
        // out of memory!
        //
        // It would be good, here, to call error_exit from util.h.
        // Doing so, however, would be the only thing which connects
        // this module to S7, which in turn requires that
        // test/test-unicode.c include a definition of global S7.
        // That's not a terrible thing, but makes the overall network
        // of dependencies that bit more complicated, and in turn
        // slightly complicates the linking step in Linux (suddenly
        // GCC needs to include -lm for some reason!), so this seems
        // something we should avoid until there's a more pressing
        // need for it than this.
        fprintf(stderr, "Out of memory creating error message (%s) in unicode.c", fmt);
        return NULL;
    }
    va_end(ap);
    return msg;
}
#endif

// Returned by decode_utf8 at EOF
// (not a valid codepoint)
const codepoint_t UNICODE_EOF = -1;
// Returned by decode_utf8 when it can't decode a UTF-8 sequence
// (also not a valid codepoint).
const codepoint_t UNICODE_BAD_DECODE = -2;
const codepoint_t UNICODE_REPLACEMENT_CHARACTER = 0xfffdU;

#if HAVE_ICU
inline int is_surrogate(const codepoint_t cp)
{
    return U16_IS_SURROGATE(cp);
}

inline codepoint_t from_surrogate(const uint16_t* const p, const char** errmsg)
{
    if (U16_IS_LEAD(p[0]) && U16_IS_TRAIL(p[1])) {
        return U16_GET_SUPPLEMENTARY(p[0], p[1]);
    } else {
        if (errmsg != NULL) *errmsg = create_errmsg("from_surrogate: bad call with U+%04x U+%04x", p[0], p[1]);
        return UNICODE_BAD_DECODE;
    }
}

codepoint_t decode_utf8(const unsigned char* bc,
                        size_t len,
                        unsigned char* nused,
                        const char** errmsg)
{
    int i = 0;
    codepoint_t rval;

    U8_NEXT(bc, i, len, rval);

    if (nused != NULL) *nused = i;
    if (rval < 0) {
        if (errmsg != NULL) *errmsg = "can't decode UTF-8";
        rval = UNICODE_BAD_DECODE;
    }
    return rval;
}

const unsigned char* encode_utf8(const codepoint_t cp, int* len)
{
    static unsigned char b[5];
    int idx = 0;
    char is_error = 0;
    unsigned char* rval;

    assert (len != NULL);

    U8_APPEND(b, idx, 5, cp, is_error);

    if (is_error) {
        *len = 0;
        rval = NULL;
    } else {
        // the function isn't documented to return a NULL-terminated
        // string, but it seems wise to do so nonetheless
        b[idx] = '\0';
        *len = idx;
        rval = b;
    }
    return rval;
}

#else
// Tests whether a codepoint (as opposed to a UTF-16 code unit) is a surrogate.
inline int is_surrogate(const codepoint_t cp)
{
    return (cp & 0xf800) == 0xd800;
    // equivalent to the following (but a whole opcode faster! -- wheee!)
    //return cp >= 0xd800 && cp < 0xe000;
}

// Convert two bytes, containing UTF-16 surrogates, to a single codepoint.
// The argument p is taken to be a two-element array.
// We don't do any error-checking here (cf, RFC 2781, Sect.2.2),
// because we presume that the content here was written only
// by other functions in this module.
// Since we don't protect the relevant memory, this means, of course,
// that a mischievous client could interfere with the content of the
// 's' field of the ustring_t object, producing potentially unwanted
// effects.  I don't plan to worry about this just now.
static inline codepoint_t from_surrogate0(const uint16_t* const p)
{
    return ((p[0] & 0x3ff) << 10 | (p[1] & 0x3ff)) + 0x10000;
}

codepoint_t from_surrogate(const uint16_t* const p, const char** errmsg)
{
    codepoint_t rval;
    if (p[0] >= 0xd800 && p[0] < 0xdc00
        && p[1] >= 0xdc00 && p[1] < 0xe000) {
        rval = from_surrogate0(p);
    } else {
        if (errmsg != NULL) *errmsg = "bad surrogate pair!";
        rval = UNICODE_BAD_DECODE;
    }
    return rval;
}

// Decode a UTF-8 sequence of length 'len', returning a single codepoint.
//
// The buffer bc is of length len (which must be greater than 0).
// This must be at least as long as the  number of bytes implied by
// the byte bc[0].  If nused is not NULL, then *nused is set to the
// number of bytes converted, which will always be at least 1.
//
// If there is an error, then return UNICODE_BAD_DECODE
// (not zero, since that's a valid codepoint which _might_ appear in some contexts).
// If additionally errmsg is not NULL, then set *errmsg to a pointer
// to an error message, which must be freed by the caller.
//
// On return (and if errmsg != NULL), either
// (the return value is non-zero) XOR (*errmsg is non-zero).
//
// It should be the case that any non-zero codepoint that this
// function returns is a codepoint that encode_utf8 will encode
// without error.
//
// If nused is not NULL, then on return (1 <= *nused <= 4)
//
// Compare https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
codepoint_t decode_utf8(const unsigned char* bc, size_t len,
                        unsigned char* nused, const char** errmsg)
{
    const unsigned char* b = (const unsigned char*)bc;
    unsigned int nbytes = 0;
    codepoint_t rval = UNICODE_BAD_DECODE;

    if (errmsg != NULL) *errmsg = NULL; // default

    if (len < 1) {
        // this is an internal function: should this perhaps be an assertion instead?
        if (errmsg != NULL) *errmsg = create_errmsg("bad call to decode_utf8: len<1");
        // ensure *nused is >0 on exit (artificial, but this conforms
        // to the guarantee in the function contract)
        nbytes = 1;
        goto bail_out;
    }

    if ((b[0] & 0x80) == 0) { // 0xxxxxxx
        nbytes = 1;
    } else if ((b[0] & 0xe0) == 0xc0) { // 110xxxxx
        nbytes = 2;
    } else if ((b[0] & 0xf0) == 0xe0) { // 1110xxxx
        nbytes = 3;
    } else if ((b[0] & 0xf8) == 0xf0) { // 11110xxx
        nbytes = 4;
    } else {
        if (errmsg != NULL) *errmsg = create_errmsg("invalid UTF-8 byte-0: 0x%x", b[0]);
        nbytes = 1;
        goto bail_out;
    }

#if 0
    // or verbosity >2?
    fprintf(stderr, "decoding [%zu]", len);
    for (int i=0; i<len; i++) {
        fprintf(stderr, " 0x%x", b[i]);
    }
    fprintf(stderr, ", nbytes=%d\n", nbytes);
#endif

    if (nbytes > len) {
        if (errmsg != NULL) *errmsg = create_errmsg("malformed UTF-8 byte 0x%x indicates more bytes than are available", b[0]);
        nbytes = len;
        goto bail_out;
    }

    for (int i=nbytes-1; i>0; i--) {
        if ((b[i] & 0xc0) != 0x80) {
            if (errmsg != NULL) *errmsg = create_errmsg("malformed UTF-8: bad continuation byte %d: 0x%x", i, b[i]);
            goto bail_out;
        }
    }

    // Check for overlong sequences -- that is, sequences which have
    // used more bytes to encode them than necessary.
    // Rather than checking for giveaway bit patterns, just base this
    // on the largest codepoints correctly encoded to a given length.
    //
    // Such sequences are not ill-formed as such (if I'm interpreteting things
    // correctly, but see the Unicode spec, ch.3, which makes some
    // rather opaque remarks on these lines), but they are non-minimal and
    // (according to Kuhn) should be rejected on security grounds.
    int overlong = 0;

    switch (nbytes) {
      case 1:
        rval = b[0];
        break;
      case 2:
        rval = ((b[0] & 0x1f) << 6) | (b[1] & 0x3f);
        overlong = (rval < 0x80);
        break;
      case 3:
        rval = ((b[0] & 0x0f) << 12) | ((b[1] & 0x3f) << 6) | (b[2] & 0x3f);
        overlong = (rval < 0x800);
        break;
      default:                  // case 4
        rval = ((b[0] & 0x07) << 18) | ((b[1] & 0x3f) << 12) | ((b[2] & 0x3f) << 6) | (b[3] & 0x3f);
        overlong = (rval < 0x10000);
    }

    // check rval...
    // (it might make sense to adjust these to use functions from
    // mycu.c, and prohibit characters of category Cx).
    if (overlong) {
        if (errmsg != NULL) *errmsg = create_errmsg("UTF-8 overlong sequence: U+%x encoded in %d bytes", rval, nbytes);
        rval = UNICODE_BAD_DECODE;
    } else if (is_surrogate(rval)) {
        if (errmsg != NULL) *errmsg = create_errmsg("UTF-8 decodes to surrogate: U+%x", rval);
        rval = UNICODE_BAD_DECODE;
    } else if (rval > 0x10ffff) {
        if (errmsg != NULL) *errmsg = create_errmsg("UTF-8 sequence out of range: U+%x", rval);
        rval = UNICODE_BAD_DECODE;
    }

 bail_out:
    assert(nbytes > 0);
    if (nused != NULL) *nused = nbytes;

    // error return behaviour (assuming errmsg is not NULL):
    // either we have a successful return and *errmsg is NULL,
    // or     we have a failure return and *errmsg is non-NULL
#if 0
    if (! ((errmsg == NULL)
           || (rval == UNICODE_BAD_DECODE && *errmsg != NULL)
           || (rval != UNICODE_BAD_DECODE && *errmsg == NULL))) {
        fprintf(stderr, "ERROR: bc=");
        for (int i=0; i<len; i++) fprintf(stderr, " %x", bc[i]);
        fprintf(stderr, ": rval=%d and *errmsg is %s (%p)\n",
                rval, (*errmsg == NULL ? "<NULL>" : *errmsg), errmsg);
    }
#endif
    assert((errmsg == NULL)
           || (rval == UNICODE_BAD_DECODE && *errmsg != NULL)
           || (rval != UNICODE_BAD_DECODE && *errmsg == NULL));

    return rval;
}

// Encode a unicode codepoint into a static buffer, and return a
// pointer to it, writing into *len the number of bytes thus encoded.
// That is, this is not thread-safe.
//
// If the codepoint is a surrogate or out of the Unicode range,
// then return NULL.
//
// len must be non-NULL.
// This is an internal function:
// if len is NULL, then produce an assertion error.
const unsigned char* encode_utf8(const codepoint_t codepoint, int* len)
{
    assert (len != NULL);

    static unsigned char b[5];
    unsigned char* rval = b;

    if (codepoint < 0x80) {
        *len = 1;
        b[0] = codepoint;
    } else if (codepoint <= 0x7ff) {
        *len = 2;
        unsigned int y = (codepoint & 0x07c0) >> 6;
        unsigned int z = codepoint & 0x3f;
        b[0] = 0xc0 | y;
        b[1] = 0x80 | z;
    } else if (is_surrogate(codepoint)) {
        // this is a UTF-16 surrogate pair character
        *len = 0;
        rval = NULL;
    } else if (codepoint <= 0xffff) {
        *len = 3;
        unsigned int x = (codepoint & 0xf000) >> 12;
        unsigned int y = (codepoint & 0x0fc0) >> 6;
        unsigned int z = (codepoint & 0x3f);
        b[0] = 0xe0 | x;
        b[1] = 0x80 | y;
        b[2] = 0x80 | z;
    } else if (codepoint <= 0x10ffff) {
        *len = 4;
        unsigned int w = (codepoint & 0x1c0000) >> 18;
        unsigned int x = (codepoint & 0x03f000) >> 12;
        unsigned int y = (codepoint & 0x0fc0)   >> 6;
        unsigned int z = (codepoint & 0x3f);
        b[0] = 0xf0 | w;
        b[1] = 0x80 | x;
        b[2] = 0x80 | y;
        b[3] = 0x80 | z;
    } else {
        // out of range
        *len = 0;
        rval = NULL;
    }

    // the function isn't documented to return a NULL-terminated
    // string, but it seems wise to do so nonetheless
    b[*len] = '\0';

    return rval;
}

#endif // !HAVE_ICU

// assert the _initial_ state of the reader
static void assert_unicode_reader_valid(const unicode_reader* r)
{
    // if 'in' is NULL, then buf must be too, and us must be non-NULL
    if (r->in == NULL) {        // reading from a ustring
#if !HAVE_ICU
        assert(r->buf == NULL);
#endif
        assert(r->us != NULL);
        assert(r->filename == NULL);
    } else {                    // reading from a file
#if !HAVE_ICU
        assert(r->buf != NULL);
#endif
        assert(r->us == NULL);
        assert(r->filename != NULL);
    }

    // it's OK for r->buflen to be zero, for a zero-length input
    assert(r->idx == 0);

    assert(r->count == 0);
    assert(r->line_count == 1);
    assert(r->ascii_p == 0 || r->ascii_p == 1);

    // idx, count and line_count will change from these
    // values as the reader is used
}

// Create a unicode_reader from a file.
// We assume, currently, that the file contents are encoded in UTF-8.
unicode_reader* make_unicode_reader_file(const char* filename,
                                         size_t buflen,
                                         const char** errmsg)
{
    unicode_reader* p = NULL;

    if ((p = malloc(sizeof(unicode_reader))) == NULL) {
        if (errmsg != NULL) {
            *errmsg = create_errmsg("make_unicode_reader_file: can't allocate memory");
        }
        goto errexit;
    }
    // initialise to safe values,
    // in case we go to errexit and thence to unicode_reader_free early
    p->in = NULL;
    p->filename = NULL;
    p->us = NULL;

#if HAVE_ICU
    if (filename) {
        if ((p->in = u_fopen(filename, "r",
                             NULL, // locale
                             "UTF_8")) == NULL) { // only UTF-8 for the mo
            if (errmsg != NULL) {
                *errmsg = create_errmsg("make_unicode_reader_file: can't open file <%s> (%s)",
                                        filename, strerror(errno));
            }
            goto errexit;
        }
    } else {
        p->in = u_fadopt(stdin, NULL, "UTF_8");
        filename = "*stdin*";
    }
#else
    p->buf = NULL;  // safe initialiser
    if (filename) {
        if ((p->in = fopen(filename, "r")) == NULL) {
            if (errmsg != NULL) {
                *errmsg = create_errmsg("make_unicode_reader_file: can't open file <%s> (%s)",
                                        filename, strerror(errno));
            }
            goto errexit;
        }
    } else {
        p->in = stdin;
        filename = "*stdin*";
    }
#endif
    p->us = NULL;

    size_t fnlen = strlen(filename);
    if ((p->filename = malloc(fnlen+1)) == NULL) {
        if (errmsg != NULL) {
            *errmsg = create_errmsg("make_unicode_reader_file: can't allocate filename buffer with %zu bytes", fnlen);
        }
        goto errexit;
    }
    memcpy(p->filename, filename, fnlen);
    p->filename[fnlen] = '\0';

#if !HAVE_ICU
    if ((p->buf = malloc(buflen)) == NULL) {
        if (errmsg != NULL) {
            *errmsg = create_errmsg("make_unicode_reader_file: can't allocate buffer of %zu bytes", buflen);
        }
        goto errexit;
    }
    p->buflen = buflen;

    // the initial values of len and idx don't matter,
    // as long as they're equal
    p->len = 0;
#endif

    p->idx = p->count = 0;
    p->line_count = 1;          // start on line 1
    p->ascii_p = 0;
    p->pushed = 0;
    p->at_eof_p = 0;

    assert_unicode_reader_valid(p);

    return p;

 errexit:
    unicode_reader_free(p);
    return NULL;
}

// It might make more sense for this string argument to be unsigned char,
// since it's intended to be bytes, but this function is in fact more
// typically used for debugging and testing, where char* is more
// convenient, so stick with that for now.
//
// We assume, currently, that the string is encoded in UTF-8.
unicode_reader* make_unicode_reader_string(const unsigned char* bstr, const char** errmsg)
{
    ustring_t us = make_ustring(errmsg);
    if (us == NULL) goto errexit;

    if (ustring_append_utf8(us, bstr, errmsg) == NULL) goto errexit;

    unicode_reader* ur = make_unicode_reader_ustring(us, errmsg);
    if (ur == NULL) goto errexit;

    // tell ur to take ownership of this ustring
    ur->our_ustring_p = 1;

    assert_unicode_reader_valid(ur);
    return ur;

  errexit:
    if (us != NULL) ustring_free(us);
    return NULL;
}

// Make a unicode_reader which takes data from the given ustring.
// We do not take ownership of this string,
// in the sense that our_ustring_p is left false.
unicode_reader* make_unicode_reader_ustring(const ustring_t us, const char** errmsg)
{
    unicode_reader* p = NULL;
    if ((p = malloc(sizeof(unicode_reader))) == NULL) {
        if (errmsg != NULL) {
            *errmsg = create_errmsg("make_unicode_reader_ustring: can't allocate memory");
        }
        goto errexit;
    }

#if !HAVE_ICU
    p->buf = NULL;
    p->buflen = p->len = us->len_;
#endif
    p->in = NULL;
    p->us = us;
    p->filename = NULL;

    p->idx = 0;
    p->our_ustring_p = 0;

    p->count = 0;
    p->pushed = 0;
    p->line_count = 1;
    p->ascii_p = p->at_eof_p = 0;

    assert_unicode_reader_valid(p);

    return p;

 errexit:
    unicode_reader_free(p);
    return NULL;
}

void unicode_reader_free(unicode_reader* p)
{
    if (p == NULL) return;

    // gcc gets upset about testing p->in uninitialised (silly gcc);
    // llvn doesn't like this warning, so complains about that (pernickety llvm)
    // (note: llvm does define __GNUC__, but gcc doesn't define __clang__)
#pragma GCC diagnostic push
#if __GNUC__ && !__clang__
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif

#if HAVE_ICU
    if (p->in != NULL) u_fclose(p->in);
    p->in = NULL;
#else
    if (p->in != NULL) fclose(p->in);
    p->in = NULL;
    if (p->buf != NULL) free(p->buf);
    p->buf = NULL;
#endif
    if (p->us != NULL && p->our_ustring_p) ustring_free(p->us);
    if (p->filename != NULL) free(p->filename);
    p->filename = NULL;

#pragma GCC diagnostic pop

    free(p);
}

// Indicate the 'source' of a reader, by writing it in to a buffer.
// This is a filename, or a "string".
// If is_file_p is not NULL, then it is set to true if the source is a
// file, or false if a string.
// Returns the number of characters written to the buffer.
const size_t unicode_reader_get_source(unicode_reader* ur,
                                       char* buf,
                                       size_t buflen,
                                       char* is_file_p)
{
    char* str;
    char free_str_p = 0;

    if (ur->filename) {
        if (is_file_p != NULL) *is_file_p = 1;
        str = ur->filename;
        free_str_p = 0;
    } else {
        if (is_file_p != NULL) *is_file_p = 0;
        str = (char*)ustring_to_utf8(ur->us, NULL);
        free_str_p = 1;
    }

    // the following is a bit clumsy, since it might cut off a UTF-8
    // sequence (no major harm; maybe come back to this later...)
    char* endp = stpncpy(buf, str, buflen-1);
    buf[buflen-1] = '\0';         // in case str is exactly buflen-1 long
    if (free_str_p) free(str);

    return endp - buf;
}

/* line Feed, vertical tab, form feed, carriage return,
 * next line, line separator, paragraph separator */
#define NEWLINE(c) ( \
	((c) >= 0xa && (c) <= 0xd) || \
	(c) == 0x85 || (c) == 0x2028 || (c) == 0x2029 )

#define DEBUG_READ 0
const codepoint_t unicode_reader_next_cp(unicode_reader* p, const char** errmsg)
{
    codepoint_t cp;

    if (p->at_eof_p) {
        cp = UNICODE_EOF;

    } else if (p->pushed) {
        cp = p->pushed;
        p->pushed = 0;

    } else if (p->in) {
        // reading from a file
#if HAVE_ICU
        cp = u_fgetcx(p->in);
#if DEBUG_READ
        fprintf(stderr, "read: U+%x = %c\n", cp, cp);
#endif
        if (cp == U_EOF) {
            p->at_eof_p = 1;
            u_fclose(p->in);
            p->in = NULL;
            cp = UNICODE_EOF;
        }
#else // !HAVE_ICU
        int nleft = p->len - p->idx;
        if (nleft < 4) {
            // We want there to be always at least four bytes in the
            // buffer, so decode_utf8 has all the bytes it will need.
            // The only time this won't be true is if we read fewer
            // than 4 bytes from the input, presuably at EOF
            if (nleft > 0) memmove(p->buf, &p->buf[p->idx], nleft);
            size_t nread = fread(&p->buf[nleft], 1, p->buflen - nleft, p->in);
            // if (nread == 0) {
            //     fclose(p->in);
            //     p->at_eof_p = 1;
            //     p->in = NULL;
            // }
#if DEBUG_READ
            fprintf(stderr, "  nread=%zu -> [%.*s]\n", nread, (int)nread+nleft, p->buf);
#endif
            p->len = nread + nleft;
            p->idx = 0;
        }

#if DEBUG_READ > 1
        fprintf(stderr, "                  -> p idx=%zu, len=%zu\n",  p->idx, p->len);
#endif

        if (p->idx >= p->len) {
            p->at_eof_p = 1;
            fclose(p->in);
            p->in = NULL;
            return UNICODE_EOF; // JUMP OUT
        }

        unsigned char nused;
        cp = decode_utf8(&p->buf[p->idx], p->len, &nused, errmsg);
        assert(nused > 0 && nused <= 4);
#if DEBUG_READ > 1
        fprintf(stderr, "  -> cp U+%x = %c (nused=%d)\n", cp, cp, nused);
#elif DEBUG_READ > 0
        fprintf(stderr, "read: U+%x = %c\n", cp, cp);
#endif
        p->idx += nused;
#endif // HAVE_ICU

        // very simple-minded line-counting! (this double counts \r\n)
        if (NEWLINE(cp)) p->line_count++;

        // we've read one codepoint, represented in nused bytes
        p->count++;

     } else {
        // reading from ustring
        if (p->idx >= p->us->idx_) {
            p->at_eof_p = 1;
            cp = UNICODE_EOF;
        } else if (is_surrogate(p->us->s_[p->idx])) {
            cp = from_surrogate(p->us->s_ + p->idx, errmsg);
            p->idx += 2;
        } else {
            cp = p->us->s_[p->idx];
            p->idx++;
        }
        p->count++;
    }

    return cp;
}

void push_codepoint(unicode_reader* p, codepoint_t cp)
{
    // if there is already a codepoint pushed, then simply overwrite it,
    // without signalling an error (provisionally)
    p->pushed = cp;
}

#define VALID_USTRING(us) assert(us != NULL                     \
                                 && us->s_ != NULL              \
                                 && us->alloc_ > 0              \
                                 && us->len_ <= us->idx_        \
                                 && us->idx_ <= us->alloc_)

// Create a new ustring object.
// If memory can't be allocated,
// then save an error message to *errmsg (if it is non-NULL),
// and return NULL.
ustring_t make_ustring(const char** errmsg)
{
    ustring_t us = malloc(sizeof(struct ustring_s));
    if (us == NULL) {
        if (errmsg != NULL) *errmsg = create_errmsg("make_ustring: can't allocate space for make_ustring");
        return NULL;
    }
    us->len_ = us->idx_ = 0;
    us->alloc_ = 16;
    ustring_cache_store_set(us, NULL);
    if ((us->s_ = malloc(us->alloc_ * sizeof(UChar))) == NULL) {
        if (errmsg != NULL) *errmsg = create_errmsg("make_ustring: can't allocate space for s_");
        free(us);
        return NULL;
    }

    VALID_USTRING(us);

    return us;
}

void ustring_free(ustring_t us)
{
    if (us == NULL) return;

    // for pragma notes, see above
#pragma GCC diagnostic push
#if __GNUC__ && !__clang__
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif

    if (us->s_ != NULL) free(us->s_);
    us->s_ = NULL;
    ustring_cache_store_set(us, NULL);

#pragma GCC diagnostic pop

    free(us);
}

// Reset a ustring's contents, without doing any allocations or deallocations
void ustring_reset(ustring_t us)
{
    us->idx_ = us->len_ = 0;
    ustring_cache_store_set(us, NULL);

    VALID_USTRING(us);
}

int ustring_length(ustring_t us)
{
    return us->len_;
}

// Add a cache to ustrings.
//
// This is implemented in ustrings, but used by higher layers to cache
// things derived from a ustring.  The cache has to be implemented there,
// so that it can be invalidated on any changes to the ustring.
//
// Yes, this slightly smacks of premature optimisation, but heads off the
// recurring temptation to expose bstrings to the rest of the code (and in
// particular the substantial attempt to use those within author-list
// parsing, backed out from), by making it reasonable to pass ustrings back
// and forth and implicitly reparse them.
const void* ustring_cache_store_get(ustring_t us)
{
    return us->cache_store_;
}

void ustring_cache_store_set(ustring_t us, const void* store)
{
    us->cache_store_ = store;
}

//// Collation and locales

#if HAVE_ICU
static char* current_locale_ = NULL;
static UCollator* current_collator_ = NULL;

// Determine if two ustrings are equal, in a locale-sensitive way.
// This could be made more sophisticated -- what does and doesn't
// count as equal in (for example) normalisation terms.  A whole world
// of complication opens up...
//
// This doesn't support the USTRING_EQUAL_COLLAPSE_REPLACEMENTS flag
// that the non-ICU version does.
int ustring_equal(ustring_t us1, ustring_t us2, byte_t flags)
{
    return ucol_strcoll(current_collator_,
                        us1->s_, us1->idx_,
                        us2->s_, us2->idx_) == UCOL_EQUAL;
}

int ustring_lt(ustring_t us1, ustring_t us2)
{
    return ucol_strcoll(current_collator_,
                        us1->s_, us1->idx_,
                        us2->s_, us2->idx_) == UCOL_LESS;
}

const char* unicode_get_locale(void)
{
    return current_locale_;
}

// Set a new locale.
// Return non-zero on error.
// On error, and if errmsg is not NULL, then *errmsg is filled with an error message
// which should be freed by the caller.
int unicode_set_locale(const char* locale, char** errmsg)
{
    UErrorCode status = U_ZERO_ERROR;
    char normalised_locale[ULOC_FULLNAME_CAPACITY];
    uloc_getName(locale, normalised_locale, ULOC_FULLNAME_CAPACITY, &status);
    if (U_FAILURE(status)) {
        if (errmsg == NULL) {
            fprintf(stderr,
                    "Can't get uloc_getName: %s\n", u_errorName(status));
        } else {
            const size_t buflen = 64;
            *errmsg = malloc(buflen);
            snprintf(*errmsg, buflen,
                     "Can't get uloc_getName: %s\n", u_errorName(status));
        }
        return 1;
    }

    // The documentation at https://unicode-org.github.io/icu/userguide/locale/
    // suggests that uloc_canonlicalize does 'level 2'
    // canonicalisation, specifically mentioning "C" -> POSIX as one
    // of the possible changes.  But this specific one doesn't seem to
    // happen in fact.  I'm not sure if I'm using this incorrectly.
    // char canonicalised_locale[ULOC_FULLNAME_CAPACITY];
    // uloc_canonicalize(locale, canonicalised_locale, ULOC_FULLNAME_CAPACITY, &status);
    // fprintf(stderr, "locales: %s -> %s -> %s\n",
    //         locale, normalised_locale, canonicalised_locale);

    UCollator* new_collator = ucol_open(normalised_locale, &status);

    int rval;

    // TODO: implement some sort of stack
    if (new_collator != 0) {
        if (current_collator_ != NULL) ucol_close(current_collator_);
        current_collator_ = new_collator;
        if (current_locale_ != NULL) free(current_locale_);
        current_locale_ = strdup(normalised_locale);

        // for debugging...
        UErrorCode errorCode = U_ZERO_ERROR;
        const char* actual_locale = ucol_getLocaleByType(current_collator_,
                                                         ULOC_VALID_LOCALE,
                                                         &errorCode);
        if (verbosity_ > 1) {
            fprintf(stderr, "unicode_set_locale: set locale to %s (normalised %s; valid %s",
                    locale, normalised_locale, actual_locale);
            actual_locale = ucol_getLocaleByType(current_collator_,
                                                 ULOC_ACTUAL_LOCALE,
                                                 &errorCode);
            fprintf(stderr, "; actual %s)\n", actual_locale);

            // How to get information about locales...
            // UChar buf[ULOC_FULLNAME_CAPACITY];
            // char obuf[ULOC_FULLNAME_CAPACITY];
            // uloc_getDisplayLanguage(normalised_locale, NULL,
            //                         buf, ULOC_FULLNAME_CAPACITY,
            //                         &status);
            // fprintf(stderr, "    language: %s\n",
            //         u_strToUTF8(obuf, ULOC_FULLNAME_CAPACITY, NULL,
            //                     buf, -1, &errorCode));
            // uloc_getDisplayVariant(normalised_locale, NULL,
            //                        buf, ULOC_FULLNAME_CAPACITY,
            //                        &status);
            // fprintf(stderr, "    variant: %s\n",
            //         u_strToUTF8(obuf, ULOC_FULLNAME_CAPACITY, NULL,
            //                     buf, -1, &errorCode));
            // fprintf(stderr, "    keywords:\n");
            // UEnumeration* key_enum = uloc_openKeywords(normalised_locale, &status);
            // if (key_enum == NULL) {
            //     fprintf(stderr, "        none\n");
            // } else {
            //     const char* key;
            //     while ((key = uenum_next(key_enum, NULL, &status)) != NULL) {
            //         fprintf(stderr, "        %s\n", key);
            //     }
            //     uenum_close(key_enum);
            // }
        }

        rval = 0;

    } else {
        if (errmsg == NULL) {
            // best we can do...
            fprintf(stderr, "Unable to create a ICU collator!: %s", u_errorName(status));
        } else {
            const size_t buflen = 64; // guess; probably big enough
            *errmsg = malloc(buflen);
            snprintf(*errmsg, buflen,
                     "Unable to create a ICU collator!: %s", u_errorName(status));
        }
        rval = 1;
    }

    return rval;
}

#else

static int ustring_equal_collapse_replacements(ustring_t s1, ustring_t s2)
{
    int result = -1;
    size_t i1, i2;

    for (i1=0, i2=0; result<0; i1++, i2++) {
        if (i1 == s1->len_ || i2 == s2->len_) {
            // at the end of one of the strings
            result = (i1 == s1->len_ && i2 == s2->len_);
        } else {
            if (s1->s_[i1] == UNICODE_REPLACEMENT_CHARACTER) {
                while (i1+1 < s1->len_ && s1->s_[i1+1] == UNICODE_REPLACEMENT_CHARACTER) i1++;
            }
            if (s2->s_[i2] == UNICODE_REPLACEMENT_CHARACTER) {
                while (i2+1 < s2->len_ && s2->s_[i2+1] == UNICODE_REPLACEMENT_CHARACTER) i2++;
            }
            if (s1->s_[i1] != s2->s_[i2]) {
                result = 0;
            }
        }
    }
    return result;
}

// The only currently recognised flag is
// USTRING_EQUAL_COLLAPSE_REPLACEMENTS.  If set, this collapses
// sequences of replacement characters so "aXXXb" and "aXb" would test
// as equal (because both are legitimate responses to an invalid
// sequence).  This is mostly to support tests.
//
// More flags might emerge, as we become more Unicode-sensitive.
int ustring_equal(ustring_t s1, ustring_t s2, byte_t flags)
{
    int result;
    if (flags & USTRING_EQUAL_COLLAPSE_REPLACEMENTS) {
        result = ustring_equal_collapse_replacements(s1, s2);

    } else if (s1->len_ == s2->len_
        && s1->idx_ == s2->idx_) {
        result = 1;
        for (size_t i=0; i<s1->idx_; i++) {
            if (s1->s_[i] != s2->s_[i]) {
                //fprintf(stderr, "  comparing failed at idx %zu: %d != %d\n", i, s1->s[i], s2->s[i]);
                result = 0;
                break;
            }
        }

    } else {
        result = 0;
    }
    return result;
}

// Return 1 if the first ustring is ordered before the second
// Note: this comparison is _not_ Unicode-sensitive!
int ustring_lt(ustring_t us1, ustring_t us2)
{
    int i=0;
    uint16_t* cp1 = us1->s_;
    uint16_t* cp2 = us2->s_;

    while (1) {
        if (i == us1->idx_) return (us1->idx_ < us2->idx_);
        if (i == us2->idx_) return 0;

        if (*cp1 == *cp2) {
            cp1++;
            cp2++;
            i++;
        } else {
            return (*cp1 < *cp2);
        }
    }
}

int unicode_set_locale(const char* locale, char** errmsg)
{
    // no-op
    return 0;
}

const char* unicode_get_locale(void)
{
    return "";
}
#endif

//// Other string functions

// Given a ustring_t, call the function f() on each codepoint.
// The function will only be called on BMP codepoints, and must map BMP to BMP.
//
// Given that restriction, and the fact we want to map this function in place,
// it seems simpler to implement this on the underlying UChar string.
ustring_t ustring_map_func(ustring_t us, codepoint_t (*f)(codepoint_t))
{
    for (size_t i=0; i<us->len_; i++) {
        if (is_surrogate(us->s_[i])) {
            i++;
        } else {
            us->s_[i] = (*f)(us->s_[i]);
        }
    }
    ustring_cache_store_set(us, NULL);

    return us;
}


// Append a codepoint to the string, and return the ustring argument.
// If the codepoint is not a unicode character
// (ie, greater than U+10FFFF,
// or is a surrogate,
// or is the last two codepoints in a plane (ie, U+_fffe or U+_ffff),
// or in range U+fdd0..U+fdef ('intended for process-internal uses'),
// or if we can't allocate memory,
// then save an error message to *errmsg (if it is non-NULL) and return NULL.
ustring_t ustring_append_cp(ustring_t us, codepoint_t cp, const char** errmsg)
{
    VALID_USTRING(us);

    if (us->idx_ > us->alloc_-2) {
        us->alloc_ *= 2;
        if ((us->s_ = realloc(us->s_, us->alloc_*sizeof(uint16_t))) == NULL) {
            if (errmsg != NULL) *errmsg = create_errmsg("Can't realloc ustring to %z units", us->alloc_);
            return NULL;
        }
    }
#if HAVE_ICU
    if (U_IS_UNICODE_CHAR(cp)) {
        U16_APPEND_UNSAFE(us->s_, us->idx_, cp);
    } else {
        if (errmsg != NULL) *errmsg = create_errmsg("ustring_append_cp: can't add out-of-range codepoint U+%x (ignored)", cp);
        return NULL;
    }

#else
    if (! U_IS_UNICODE_CHAR(cp)) {
        if (errmsg != NULL) *errmsg = create_errmsg("ustring_append_cp: can't add non-character U+%x (ignored)", cp);
        return NULL;
    } else if (cp < 0x10000) {
        us->s_[us->idx_++] = cp;
    } else {
        uint32_t cpx = cp - 0x10000;
        us->s_[us->idx_]   = 0xd800 | (cpx >> 10);
        us->s_[us->idx_+1] = 0xdc00 | (cpx & 0x3ff);
        us->idx_ += 2;
    }
#endif
    us->len_++;
    ustring_cache_store_set(us, NULL);

    VALID_USTRING(us);

    return us;
}

// Append a UTF-8 encoded string to the ustring.
// Return the us argument on success, or NULL
// (with an error in *errmsg) on error.
ustring_t ustring_append_utf8(ustring_t us, const byte_t* s, const char** errmsg)
{
    VALID_USTRING(us);

    size_t slen = strlen((const char*)s);
    if (slen == 0) {
        return us;
    } else {
        return ustring_append_utf8_with_length(us, s, slen, errmsg);
    }
}

ustring_t ustring_append_utf8_with_length(ustring_t us,
                                          const byte_t* s,
                                          const size_t slen,
                                          const char** errmsg)
{
    if (slen == 0) return us;

#if HAVE_ICU
    UErrorCode status = U_ZERO_ERROR;
    int32_t destLength;         // number of code units written to the destination
    (void) u_strFromUTF8WithSub(us->s_ + us->idx_,
                                us->alloc_ - us->idx_,
                                &destLength,
                                (const char*)s,
                                slen, // length of s
                                UNICODE_REPLACEMENT_CHARACTER, NULL,
                                &status);
    if (U_SUCCESS(status)) {
        // We must increment us->idx_ by destLength,
        // but increment us->len_ by the number of codepoints
        // represented, which requires us to count the number of
        // surrogates here.
        int32_t endidx = us->idx_ + destLength;
        int nsurrogate_pairs = 0;
        for (int i=us->idx_; i<endidx; i++) {
            if (is_surrogate(us->s_[i])) {
                nsurrogate_pairs++;
                i++;            // skip over the second
            }
        }
        us->idx_ += destLength;
        us->len_ += destLength - nsurrogate_pairs;
        ustring_cache_store_set(us, NULL);

    } else if (status == U_BUFFER_OVERFLOW_ERROR) {
        // destLength contains the required size
        while (us->alloc_ < us->idx_ + destLength) us->alloc_ *= 2;
        if ((us->s_ = realloc(us->s_, us->alloc_*sizeof(UChar))) == NULL) {
            if (errmsg != NULL) *errmsg = create_errmsg("ustring_append_utf8: can't realloc ustring to %z units", us->alloc_);
            return NULL;
        }
        // recurse with larger buffer
        return ustring_append_utf8_with_length(us, s, slen, errmsg);

    } else if (status == U_INVALID_CHAR_FOUND) {
        // this probably shouldn't happen, since we're using
        // u_strFromUTF8WithSub
        ustring_append_cp(us, UNICODE_REPLACEMENT_CHARACTER, errmsg);

    } else {
        if (errmsg != NULL) *errmsg = create_errmsg("ustring_append_utf8: unexpected error: %s", u_errorName(status));
        return NULL;
    }

#else
    size_t idx = 0;
    while (idx < slen) {
        unsigned char nused;
        codepoint_t cp = decode_utf8(&s[idx], slen, &nused, errmsg);
        //ustring_append_cp(us, cp, errmsg); // ...even if it's UNICODE_BAD_DECODE
        if (cp == UNICODE_BAD_DECODE) {
            ustring_append_cp(us, UNICODE_REPLACEMENT_CHARACTER, errmsg);
        } else {
            ustring_append_cp(us, cp, errmsg);
        }
        idx += nused;
    }
#endif

    VALID_USTRING(us);

    return us;
}


// Append the uchars array to the ustring.
// The argument n_cp is the number of codepoints in the string,
// and n_units is the number of code units, which will be more than
// n_cp if some of the codepoints occupy two code-units.
ustring_t ustring_append_uchars(ustring_t us1, UChar* uchars,
                                size_t n_cp, // number of codepoints in the UChar string
                                size_t n_units, // number of code units, >= n_cp
                                const char** errmsg)
{
    VALID_USTRING(us1);

#if 0
    // No: don't do this (why did I think this was needed?)
    // If either of n_cp or n_units is zero, then count to the first null
    // in uchars.
    if (n_cp == 0 || n_units == 0) {
#if HAVE_ICU
        n_units = u_strlen(uchars);
        n_cp = u_countChar32(uchars, -1);
#else
        n_units = 0;
        for (n_cp=0; uchars[n_cp] != 0; n_cp++) {
            if (is_surrogate(uchars[n_cp])) n_units++;
            n_units++;
        }
#endif
    }
#endif
    if (n_units < n_cp) {
        if (errmsg != NULL) *errmsg = create_errmsg("ustring_append_chars: bad call: fewer code-units (%d) than codepoints (%d)", n_units, n_cp);
        return NULL;
    }

    size_t newidx = us1->idx_ + n_units;
    if (newidx > us1->alloc_-2) {
        while (newidx > us1->alloc_-2) us1->alloc_ *= 2;
        if ((us1->s_ = realloc(us1->s_, us1->alloc_*sizeof(UChar))) == NULL) {
            if (errmsg != NULL) *errmsg = create_errmsg("Can't realloc ustring to %z units", us1->alloc_);
            return NULL;
        }
    }
    memcpy(&us1->s_[us1->idx_], uchars, n_units*sizeof(UChar));
    us1->idx_ += n_units;
    us1->len_ += n_cp;
    ustring_cache_store_set(us1, NULL);

    VALID_USTRING(us1);

    return us1;
}


ustring_t ustring_append_ustring(ustring_t us1, ustring_t us2, const char** errmsg)
{
    VALID_USTRING(us1);
    VALID_USTRING(us2);

    return ustring_append_uchars(us1, us2->s_, us2->len_, us2->idx_, errmsg);
}


// Return the codepoint at index in the ustring.
// Return 0 if the index is out of range.
codepoint_t ustring_ref(ustring_t us, size_t refidx)
{
    if (refidx < 0 || refidx >= us->len_) {
        return 0;
    } else if (us->len_ == us->idx_) {
        // there are no codepoints requiring two UTF-16 code units
        return us->s_[refidx];
    } else {
        uint16_t* a = us->s_;
        size_t i = 0;           // count indexes into a[]
        for (size_t charnum=0;  // count characters
             charnum<refidx;
             charnum++, i++) {
            if (is_surrogate(a[i])) i++;
        }
        if (is_surrogate(a[i])) {
            return from_surrogate(&a[i], NULL);
        } else {
            return a[i];
        }
    }
}

// Return a (newly-allocated) substring of the given string,
// starting at character index `start`, and ending at index `end`,
// or the end of the string if `end` is negative.
//
// This is a character offset, and not a code-unit offset.
//
// Returns NULL if the start is negative,
// or if end<start.
// Currently, if end is beyond the end of the string we don't report an error.
//
// In C terms, the result must be freed after use.
ustring_t ustring_substring(ustring_t us, int start, int end, const char** errmsg)
{
    // fprintf(stderr, "ustring_substring: start=%d  end=%d\n", start, end);

    if (start < 0 || (end >= 0 && end < start)) {
        if (errmsg != NULL) *errmsg = create_errmsg("ustring_substring: bad call: start=%d must be <= end=%d", start, end);
        return NULL;
    }

    int copy_cp, copy_codeunit;
    if (end < 0 || end > us->len_) {
        // silently take end beyond EOS to be 'all of the string'
        copy_cp = us->len_ - start;
        copy_codeunit = us->idx_ - start;
    } else {
        copy_cp = copy_codeunit = end - start;
        if (us->len_ != us->idx_) {
            // there are some codepoints requiring two UTF-16 code units:
            // increment copy_codeunit by the number of such characters
            for (int i=start; i<start+copy_cp; i++) {
                if (is_surrogate(us->s_[i])) {
                    copy_codeunit++;
                    i++;        // jump over the other surrogate
                }
            }
        }
    }

    // fprintf(stderr, "ustring_substring: start=%d  end=%d  copy_cp=%d  copy_codeunit=%d\n",
    //         start, end, copy_cp, copy_codeunit);

    ustring_t res = make_ustring(NULL);
    return ustring_append_uchars(res,
                                 &us->s_[start],
                                 copy_cp,       // no. codepoints
                                 copy_codeunit, // no. code units
                                 NULL);
}

// Encode the ustring in UTF-8; the result is zero-terminated.
// The returned string must be freed after use.
// If we run out of memory, then create an error message in *errmsg
// (if non-NULL) and return NULL.
#if HAVE_ICU
const byte_t* ustring_to_utf8(ustring_t us, const char** errmsg)
{
    size_t destlen = us->idx_ + us->idx_/2; // idx*1.5 (rough estimate)
    if (destlen < 2) destlen = 2; // oops, idx_==0, so ensure destlen>=2

    char* rval;
    if ((rval = malloc(destlen * sizeof(char))) == NULL) {
        if (errmsg != NULL) *errmsg = create_errmsg("ustring_to_utf8: unable to allocate %z bytes", destlen);
        return NULL;
    }

    int32_t actualDestLength = 0;
    UErrorCode status = U_ZERO_ERROR;
    // the destlen-1 is to ensure we have space to null-terminate
    // at rval[actualDestLength]
    u_strToUTF8(rval, destlen-1, &actualDestLength, us->s_, us->idx_, &status);

    if (U_FAILURE(status)) {
        if (status == U_BUFFER_OVERFLOW_ERROR) {
            // We underestimated destlen above.
            // No problem: try again with the returned actualDestLength.
            destlen = actualDestLength+1; // +1 for zero-termination
            if ((rval = realloc(rval, destlen * sizeof(char))) == NULL) {
                if (errmsg != NULL) *errmsg = create_errmsg("ustring_to_utf8: unable to reallocate %z bytes", destlen);
                return NULL;
            }
            // try again...
            status = U_ZERO_ERROR;
            u_strToUTF8(rval, destlen, &actualDestLength, us->s_, us->idx_, &status);
            if (U_FAILURE(status)) {
                if (errmsg != NULL) *errmsg = create_errmsg("ustring_to_utf8: unexpected realloc error: %s", u_errorName(status));
                return NULL;
            }

        } else {
            if (errmsg != NULL) *errmsg = create_errmsg("ustring_to_utf8: unexpected error: %s", u_errorName(status));
            return NULL;
        }
    }

    rval[actualDestLength] = '\0';

    return (byte_t*)rval;
}

#else
const byte_t* ustring_to_utf8(ustring_t us, const char** errmsg)
{
    size_t reqlen = 0;
    for (int i=0; i<us->idx_; i++) {
        if (us->s_[i] < 0x80) {
            reqlen += 1;
        } else if (us->s_[i] < 0x800) {
            reqlen += 2;
        } else if (is_surrogate(us->s_[i])) {
            // surrogate (should be a high-surrogate)
            reqlen += 4;
            i++;                // jump over low-surrogate
        } else {
            reqlen += 3;
        }
    }

    byte_t* b = malloc(reqlen + 1);
    if (b == NULL) {
        if (errmsg != NULL) *errmsg = create_errmsg("ustring_to_utf8: Unable to allocate %z bytes", reqlen);
        return NULL;
    }

    size_t bi = 0;
    size_t ui = 0;
    while (ui < us->idx_) {
        codepoint_t cp;
        if (is_surrogate(us->s_[ui])) {
            cp = from_surrogate0(&us->s_[ui]);
            ui += 2;
        } else {
            cp = us->s_[ui];
            ui++;
        }

        int len;
        const byte_t* b8 = encode_utf8(cp, &len);
        memcpy(&b[bi], b8, len);
        bi += len;
    }
    b[bi] = 0;

    assert(bi == reqlen);

    return b;
}
#endif

// Initialise this module.
// Return zero on success.
// On error, and if errmsg is not NULL, fill *errmsg with an error message,
// which must be freed afterwards.
int initialise_unicode_module(char** errmsg)
{
#if HAVE_ICU
    if (current_collator_ != NULL) return 0; // idempotent

    return unicode_set_locale(getenv("BEASTIE_LOCALE"), // env is NULL if not defined
                              errmsg);
#else
    return 0;
#endif
}