pcre/maint/utf8.c

/****************************************************
* PCRE maintainers' helper program: UTF-8 converter *
****************************************************/

/* This is a test program for converting character code points to UTF-8 and
vice versa. Note that this program conforms to the original definition of
UTF-8, which allows codepoints up to 7fffffff. The more recent definition
limits the validity of Unicode UTF-8 codepoints to a maximum of 10ffffff, and
forbids the "surrogate" code points. This program now gives warnings for these
invalid code points.

The arguments are either single code point values written as U+hh.. or 0xhh..
for conversion to UTF-8, or sequences of hex values, written without 0x and
optionally including spaces (but such arguments must be quoted), for conversion
from UTF-8 to codepoints. For example:

./utf8 0x1234
U+00001234 => e1 88 b4

./utf8 "e1 88 b4"
U+00001234 <= e1 88 b4

In the second case, a number of UTF-8 characters can be present in one
argument. In other words, each such argument is interpreted (after ignoring
spaces) as a string of UTF-8 bytes representing a string of characters:

./utf8 "65 e188b4 77"
0x00000065 <= 65
0x00001234 <= e1 88 b4
0x00000077 <= 77

If the option -s is given, the sequence of UTF-bytes is written out between
angle brackets at the end of the line. On a UTF-8 terminal, this will show the
appropriate graphic for the code point.

Errors provoke error messages, but the program carries on with the next
argument. The return code is always zero.

Philip Hazel
Original creation data: unknown
Code extended and tidied to avoid compiler warnings: 26 March 2020
*/


#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <string.h>

/* The valid ranges for UTF-8 characters are:

0000 0000  to  0000 007f   1 byte (ascii)
0000 0080  to  0000 07ff   2 bytes
0000 0800  to  0000 ffff   3 bytes
0001 0000  to  001f ffff   4 bytes
0020 0000  to  03ff ffff   5 bytes
0400 0000  to  7fff ffff   6 bytes
*/


static const unsigned int utf8_table1[] = {
  0x0000007f, 0x000007ff, 0x0000ffff, 0x001fffff, 0x03ffffff, 0x7fffffff};

static const int utf8_table2[] = {
  0, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc};

static const int utf8_table3[] = {
  0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01};


/*************************************************
*       Convert character value to UTF-8         *
*************************************************/

/* This function takes an unsigned long integer value in the range 0 -
0x7fffffff and encodes it as a UTF-8 character in 1 to 6 bytes.

Arguments:
  cvalue     the character value
  buffer     pointer to buffer for result - at least 6 bytes long

Returns:     number of bytes placed in the buffer
             0 if input code point is too big
*/

static size_t
ord2utf8(unsigned long int cvalue, unsigned char *buffer)
{
size_t i, j;
for (i = 0; i < sizeof(utf8_table1)/sizeof(int); i++)
  if (cvalue <= utf8_table1[i]) break;
if (i >= sizeof(utf8_table1)/sizeof(int)) return 0;
buffer += i;
for (j = i; j > 0; j--)
 {
 *buffer-- = 0x80 | (cvalue & 0x3f);
 cvalue >>= 6;
 }
*buffer = utf8_table2[i] | cvalue;
return i + 1;
}



/*************************************************
*            Convert UTF-8 string to value       *
*************************************************/

/* This function takes one or more bytes that represent a UTF-8 character from
the start of a string of bytes. It returns the value of the character, or the
offset of a malformation. For an overlong encoding that works but is not the
correct (shortest) one, the error offset is just after the last byte.

Argument:
  buffer   a pointer to the byte vector
  buffend  a pointer to the end of the buffer
  vptr     a pointer to a variable to receive the value
  lenptr   a pointer to a variable to receive the offset when error detected

Returns:   > 0 => the number of bytes consumed
             0 => invalid UTF-8: first byte missing 0x40 bit
            -1 => invalid UTF-8: first byte has too many high-order 1-bits
            -2 => incomplete sequence at end of string
            -3 => incomplete sequence within string
            -4 => overlong code sequence
*/

static int
utf82ord(unsigned char *buffer, unsigned char *buffend,
  long unsigned int *vptr, int *lenptr)
{
unsigned int c = *buffer++;
unsigned int d = c;
int i, j, s;

/* Check for an ASCII character, or find the number of additional bytes in a
multibyte character. */

for (i = -1; i < 6; i++)
  {
  if ((d & 0x80) == 0) break;
  d <<= 1;
  }

switch (i)
  {
  case -1:      /* ASCII character; first byte does not have 0x80 bit */
  *vptr = c;
  return 1;

  case 0:       /* First byte has 0x80 but is missing 0x40 bit */
  *lenptr = 0;
  return 0;

  case 6:
  *lenptr = 0;  /* Too many high bits */
  return -1;

  default:
  break;
  }

/* i now has a value in the range 1-5 */

s = 6*i;
d = (c & utf8_table3[i]) << s;

for (j = 0; j < i; j++)
  {
  if (buffer >= buffend)
    {
    *lenptr = j + 1;
    return -2;
    }
  c = *buffer++;
  if ((c & 0xc0) != 0x80)
    {
    *lenptr = j + 1;
    return -3;
    }
  s -= 6;
  d |= (c & 0x3f) << s;
  }

/* Valid UTF-8 syntax */

*vptr = d;

/* Check that encoding was the correct one, not overlong */

for (j = 0; j < (int)(sizeof(utf8_table1)/sizeof(int)); j++)
  if (d <= utf8_table1[j]) break;
if (j != i)
  {
  *lenptr = i + 1;
  return -4;
  }

/* Valid value */

return i + 1;
}



/*************************************************
*                 Main Program                   *
*************************************************/

int
main(int argc, char **argv)
{
int i = 1;
int show = 0;
unsigned char buffer[64];

if (argc > 1 && strcmp(argv[1], "-s") == 0)
  {
  show = 1;
  i = 2;
  }

for (; i < argc; i++)
  {
  char *x = argv[i];
  char *endptr;
  if (strncmp(x, "0x", 2) == 0 || strncmp(x, "U+", 2) == 0)
    {
    size_t rc, j;
    unsigned long int d = strtoul(x+2, &endptr, 16);
    if (*endptr != 0)
      {
      printf("** Invalid hex number %s\n", x);
      continue;   /* With next argument */
      }
    rc = ord2utf8(d, buffer);
    printf("U+%08lx => ", d);
    if (rc == 0)
      printf("** Code point greater than 0x7fffffff cannot be encoded");
    else
      {
      for (j = 0; j < rc; j++) printf("%02x ", buffer[j]);
      if (show)
        {
        printf(">");
        for (j = 0; j < rc; j++) printf("%c", buffer[j]);
        printf("< ");
        }
      if (d >= 0xd800 && d <= 0xdfff)
        printf("** Invalid Unicode (surrogate)");
      else if (d > 0x10ffff)
        printf("** Invalid Unicode (greater than U+10ffff)");
      }
    printf("\n");
    }
  else
    {
    unsigned char *bptr;
    unsigned char *buffend;
    int len = 0;
    int y = 0;
    int z = 0;

    for (;;)
      {
      while (*x == ' ') x++;
      if (*x == 0 && !z) break;
      if (!isxdigit(*x))
        {
        printf("** Malformed hex string: %s\n", argv[i]);
        len = -1;
        break;
        }
      y = y * 16 + tolower(*x) - ((isdigit(*x))? '0' : 'W');
      x++;
      if (z)
        {
        buffer[len++] = y;
        y = 0;
        }
      z ^= 1;
      }

    if (len < 0) continue;  /* With next argument after malformation */

    bptr = buffer;
    buffend = buffer + len;

    while (bptr < buffend)
      {
      unsigned long int d;
      int j;
      int offset;
      int rc = utf82ord(bptr, buffend, &d, &offset);

      if (rc > 0)
        {
        printf("U+%08lx <= ", d);
        for (j = 0; j < rc; j++) printf("%02x ", bptr[j]);
        if (show)
          {
          printf(">");
          for (j = 0; j < rc; j++) printf("%c", bptr[j]);
          printf("<");
          }
        printf("\n");
        bptr += rc;
        }
      else if (rc == -4)
        {
        printf("U+%08lx <= ", d);
        for (j = 0; j < offset; j++) printf("%02x ", bptr[j]);
        printf("** Overlong UTF-8 sequence\n");
        bptr += offset;
        }
      else
        {
        switch (rc)
          {
          case 0:  printf("** First byte missing 0x40 bit");
          break;

          case -1: printf("** First byte has too many high-order bits");
          break;

          case -2: printf("** Incomplete UTF-8 sequence at end of string");
          break;

          case -3: printf("** Incomplete UTF-8 sequence");
          break;

          default: printf("** Unexpected return %d from utf82ord()", rc);
          break;
          }
        printf(" at offset %d in string ", offset);
        while (bptr < buffend) printf("%02x ", *bptr++);
        printf("\n");
        break;
        }
      }
    }
  }

return 0;
}

/* End */