JehanneOS
/
drawterm
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

libc: import 21-bit runes from Plan 9 

R=rsc
https://codereview.appspot.com/9035046
This commit is contained in:
David du Colombier 2013-06-21 23:08:33 +02:00
parent 2f4516ba69
commit 3dbe2c359f
3 changed files with 101 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
include/lib.h
View File

@ -3,6 +3,9 @@
#define listen pm_listen #define listen pm_listen
#define sleep ksleep #define sleep ksleep
#define wakeup kwakeup #define wakeup kwakeup
#ifdef strtod
#undef strtod
#endif
#define strtod fmtstrtod #define strtod fmtstrtod
/* conflicts on some os's */ /* conflicts on some os's */
@ -27,7 +30,7 @@ typedef unsigned int p9_ulong;
typedef int p9_long; typedef int p9_long;
typedef signed char p9_schar; typedef signed char p9_schar;
typedef unsigned short p9_ushort; typedef unsigned short p9_ushort;
typedef unsigned short Rune; typedef unsigned int Rune;
typedef unsigned int p9_u32int; typedef unsigned int p9_u32int;
typedef p9_u32int mpdigit; typedef p9_u32int mpdigit;
@ -50,10 +53,12 @@ typedef p9_u32int mpdigit;
enum enum
{ {
UTFmax = 3, /* maximum bytes per rune */ UTFmax = 4, /* maximum bytes per rune */
Runesync = 0x80, /* cannot represent part of a UTF sequence (<) */ Runesync = 0x80, /* cannot represent part of a UTF sequence (<) */
Runeself = 0x80, /* rune and UTF sequences are the same (<) */ Runeself = 0x80, /* rune and UTF sequences are the same (<) */
Runeerror = 0x80 /* decoding error in UTF */ Runeerror = 0xFFFD, /* decoding error in UTF */
Runemax = 0x10FFFF, /* 21-bit rune */
Runemask = 0x1FFFFF, /* bits used by runes (see grep) */
}; };
/* /*

172
libc/rune.c
View File

@ -1,26 +1,24 @@
#include <u.h> #include <u.h>
#include <libc.h> #include <libc.h>
#define Bit(i) (7-(i))
/* N 0's preceded by i 1's, T(Bit(2)) is 1100 0000 */
#define T(i) (((1 << (Bit(i)+1))-1) ^ 0xFF)
/* 0000 0000 0000 0111 1111 1111 */
#define RuneX(i) ((1 << (Bit(i) + ((i)-1)*Bitx))-1)
enum enum
{ {
Bit1 = 7, Bitx = Bit(1),
Bitx = 6,
Bit2 = 5,
Bit3 = 4,
Bit4 = 3,
T1 = ((1<<(Bit1+1))-1) ^ 0xFF, /* 0000 0000 */ Tx = T(1), /* 1000 0000 */
Tx = ((1<<(Bitx+1))-1) ^ 0xFF, /* 1000 0000 */ Rune1 = (1<<(Bit(0)+0*Bitx))-1, /* 0000 0000 0000 0000 0111 1111 */
T2 = ((1<<(Bit2+1))-1) ^ 0xFF, /* 1100 0000 */
T3 = ((1<<(Bit3+1))-1) ^ 0xFF, /* 1110 0000 */
T4 = ((1<<(Bit4+1))-1) ^ 0xFF, /* 1111 0000 */
Rune1 = (1<<(Bit1+0*Bitx))-1, /* 0000 0000 0111 1111 */ Maskx = (1<<Bitx)-1, /* 0011 1111 */
Rune2 = (1<<(Bit2+1*Bitx))-1, /* 0000 0111 1111 1111 */ Testx = Maskx ^ 0xFF, /* 1100 0000 */
Rune3 = (1<<(Bit3+2*Bitx))-1, /* 1111 1111 1111 1111 */
Maskx = (1<<Bitx)-1, /* 0011 1111 */ SurrogateMin = 0xD800,
Testx = Maskx ^ 0xFF, /* 1100 0000 */ SurrogateMax = 0xDFFF,
Bad = Runeerror, Bad = Runeerror,
}; };
@ -28,49 +26,42 @@ enum
int int
chartorune(Rune *rune, char *str) chartorune(Rune *rune, char *str)
{ {
int c, c1, c2; int c[UTFmax], i;
long l; Rune l;
/* /*
* one character sequence * N character sequence
* 00000-0007F => T1 * 00000-0007F => T1
* 00080-007FF => T2 Tx
* 00800-0FFFF => T3 Tx Tx
* 10000-10FFFF => T4 Tx Tx Tx
*/ */
c = *(uchar*)str;
if(c < Tx) { c[0] = *(uchar*)(str);
*rune = c; if(c[0] < Tx){
*rune = c[0];
return 1; return 1;
} }
l = c[0];
/* for(i = 1; i < UTFmax; i++) {
* two character sequence c[i] = *(uchar*)(str+i);
* 0080-07FF => T2 Tx c[i] ^= Tx;
*/ if(c[i] & Testx)
c1 = *(uchar*)(str+1) ^ Tx;
if(c1 & Testx)
goto bad;
if(c < T3) {
if(c < T2)
goto bad; goto bad;
l = ((c << Bitx) | c1) & Rune2; l = (l << Bitx) | c[i];
if(l <= Rune1) if(c[0] < T(i + 2)) {
goto bad; l &= RuneX(i + 1);
*rune = l; if(i == 1) {
return 2; if(c[0] < T(2) || l <= Rune1)
} goto bad;
} else if(l <= RuneX(i) || l > Runemax)
/* goto bad;
* three character sequence if (i == 2 && SurrogateMin <= l && l <= SurrogateMax)
* 0800-FFFF => T3 Tx Tx goto bad;
*/ *rune = l;
c2 = *(uchar*)(str+2) ^ Tx; return i + 1;
if(c2 & Testx) }
goto bad;
if(c < T4) {
l = ((((c << Bitx) | c1) << Bitx) | c2) & Rune3;
if(l <= Rune2)
goto bad;
*rune = l;
return 3;
} }
/* /*
@ -84,12 +75,9 @@ bad:
int int
runetochar(char *str, Rune *rune) runetochar(char *str, Rune *rune)
{ {
long c; int i, j;
Rune c;
/*
* one character sequence
* 00000-0007F => 00-7F
*/
c = *rune; c = *rune;
if(c <= Rune1) { if(c <= Rune1) {
str[0] = c; str[0] = c;
@ -97,23 +85,35 @@ runetochar(char *str, Rune *rune)
} }
/* /*
* one character sequence
* 00000-0007F => 00-7F
* two character sequence * two character sequence
* 0080-07FF => T2 Tx * 0080-07FF => T2 Tx
*/
if(c <= Rune2) {
str[0] = T2 | (c >> 1*Bitx);
str[1] = Tx | (c & Maskx);
return 2;
}
/*
* three character sequence * three character sequence
* 0800-FFFF => T3 Tx Tx * 0800-FFFF => T3 Tx Tx
* four character sequence (21-bit value)
* 10000-1FFFFF => T4 Tx Tx Tx
* If the Rune is out of range or a surrogate half,
* convert it to the error rune.
* Do this test when i==3 because the error rune encodes to three bytes.
* Doing it earlier would duplicate work, since an out of range
* Rune wouldn't have fit in one or two bytes.
*/ */
str[0] = T3 | (c >> 2*Bitx); for(i = 2; i < UTFmax + 1; i++){
str[1] = Tx | ((c >> 1*Bitx) & Maskx); if(i == 3){
str[2] = Tx | (c & Maskx); if(c > Runemax)
return 3; c = Runeerror;
if(SurrogateMin <= c && c <= SurrogateMax)
c = Runeerror;
}
if (c <= RuneX(i) || i == UTFmax ) {
str[0] = T(i) | (c >> (i - 1)*Bitx);
for(j = 1; j < i; j++)
str[j] = Tx | ((c >> (i - j - 1)*Bitx) & Maskx);
return i;
}
}
return UTFmax;
} }
int int
@ -129,18 +129,21 @@ runelen(long c)
int int
runenlen(Rune *r, int nrune) runenlen(Rune *r, int nrune)
{ {
int nb, c; int nb, i;
Rune c;
nb = 0; nb = 0;
while(nrune--) { while(nrune--) {
c = *r++; c = *r++;
if(c <= Rune1) if(c <= Rune1){
nb++; nb++;
else } else {
if(c <= Rune2) for(i = 2; i < UTFmax + 1; i++)
nb += 2; if(c <= RuneX(i) || i == UTFmax){
else nb += i;
nb += 3; break;
}
}
} }
return nb; return nb;
} }
@ -148,15 +151,16 @@ runenlen(Rune *r, int nrune)
int int
fullrune(char *str, int n) fullrune(char *str, int n)
{ {
int c; int i;
Rune c;
if(n > 0) { if(n <= 0)
c = *(uchar*)str; return 0;
if(c < Tx) c = *(uchar*)str;
return 1; if(c < Tx)
if(n > 1) return 1;
if(c < T3 || n > 2) for(i = 3; i < UTFmax + 1; i++)
return 1; if(c < T(i))
} return n >= i - 1;
return 0; return n >= UTFmax;
} }

8
libc/utf.h
View File

@ -1,14 +1,16 @@
#ifndef _UTFH_ #ifndef _UTFH_
#define _UTFH_ 1 #define _UTFH_ 1
typedef unsigned short Rune; /* 16 bits */ typedef unsigned int Rune; /* 32 bits */
enum enum
{ {
UTFmax = 3, /* maximum bytes per rune */ UTFmax = 4, /* maximum bytes per rune */
Runesync = 0x80, /* cannot represent part of a UTF sequence (<) */ Runesync = 0x80, /* cannot represent part of a UTF sequence (<) */
Runeself = 0x80, /* rune and UTF sequences are the same (<) */ Runeself = 0x80, /* rune and UTF sequences are the same (<) */
Runeerror = 0x80, /* decoding error in UTF */ Runeerror = 0xFFFD, /* decoding error in UTF */
Runemax = 0x10FFFF, /* 21-bit rune */
Runemask = 0x1FFFFF, /* bits used by runes (see grep) */
}; };
/* /*