4 title => 'charset cheat sheet',
7 "Reference sheet with all glyphs in common character encoding tables,",
8 "and an overview of Unicode ranges and UTF-8 bytes.",
11 charset codepage unicode ascii utf8 latin glyph character encoding
12 reference common overview table
14 stylesheet => [qw'light'],
15 data => [qw'charset-unicode.inc.pl charset-utf8.inc.pl'],
19 <h1>Character encoding</h1>
22 use Shiar_Sheet::FormatChar;
23 my $glyphs = Shiar_Sheet::FormatChar->new;
24 my @nibble = (0..9, 'A'..'F');
27 use Encode qw(decode resolve_alias);
28 # generate character table(s)
29 # (~16x faster than decoding in loop;
30 # substr strings is twice as fast as splitting to an array)
32 # default => [qw(unicode utf-8 iso-8859-1 cp437 -cp1252- --iso-8859-15- -koi8-f)],
33 default => [qw(unicode- utf-8 iso-8859-1 -cp1252- --iso-8859-15- cp437 -cp850)],
34 0 => [qw(cp437 -cp863)],
35 1 => [qw(iso-8859-1 -cp1252 -MacRoman -cp850)],
36 2 => [qw(iso-8859-2 -cp1250 -cp852 -MacCentralEurRoman -MacCroatian -MacRumanian)],
37 5 => [qw(koi8-f -iso-8859-5 -cp1251 -MacCyrillic -cp855 -cp866)],
38 7 => [qw(iso-8859-7 -cp1253 -MacGreek -cp737 -cp869)],
39 8 => [qw(iso-8859-8 -cp1255 -MacHebrew -cp862)],
43 my %row = (offset => 0);
45 if ($input =~ s/^--//) {
46 $row{offset} = $endpoint > 160 ? 160 : 48;
48 elsif ($input =~ s/^-//) {
49 $row{offset} = $endpoint > 128 ? 128 : 32;
51 if ($input =~ s/-$//) {
52 $endpoint = $row{offset} ? $row{offset} < 160 ? 159 : 191 : 127;
55 $row{setnote} = 'over cp437' if $input eq 'cp850';
56 $row{setnote} = 'over iso-8859-1' if $input =~ /^iso-8859-|^cp125/;
59 if ($input =~ /^U([0-9a-f]+)(?:-([0-9a-f]+))?/) {
60 my $start = hex($1) << ($2 ? 4 : 8);
61 my $end = $2 ? hex($2) << 4 : $start + 240;
62 $row{table} = join '', map { chr } $start .. $end+15;
63 utf8::upgrade($row{table}); # prevent latin1 output
64 $row{set} = sprintf 'Unicode block U+%02Xxx', $start >> 8;
66 elsif ($input eq 'U') {
67 $row{table} = ' ' x 1024;
68 $row{set} = 'Unicode planes';
69 $row{cell} = do 'charset-ucplanes.inc.pl'
70 or printf "<p class=error>Table data could not be read: <em>%s</em>.</p>\n", $@ || $!;
71 @nibble = (map { $_.0, $_.8 } 0 .. 7);
74 elsif ($row{set} = resolve_alias($input)) {
75 if ($row{set} eq 'Internal') {
76 $row{table} = ' ' x ($endpoint < 255 ? 640 : 8192);
77 $row{set} = 'Unicode BMP';
78 $row{cell} = do 'charset-unicode.inc.pl'
79 or printf "<p class=error>Table data could not be read: <em>%s</em>.</p>\n", $@ || $!;
81 elsif ($row{set} eq 'utf-8-strict') {
84 $row{cell} = do 'charset-utf8.inc.pl'
85 or printf "<p class=error>Table data could not be read: <em>%s</em>.</p>\n", $@ || $!;
88 $row{table} = decode($row{set}, pack 'C*', $row{offset} .. $endpoint);
92 say "<p class=error>Encoding <q>$input</q> unknown</p>";
94 $row{set} ? \%row : ();
99 } map { defined $ALIAS{$_} ? @{ $ALIAS{$_} } : $_ }
100 $ENV{PATH_INFO} =~ /\w/ ? split(m{[/+\s]}, $ENV{PATH_INFO}) : 'default';
101 my $NOCHAR = chr 0xFFFD;
103 for my $cp437 (grep {$request[$_]->{set} eq 'cp437'} 0 .. $#request) {
104 substr($request[$cp437]->{table}, 237, 1) = pack 'U*', 0x3D5; # phi sign
105 substr($request[$cp437]->{table}, 0, 32) = pack 'U*', map {hex} qw(
106 2007 263A 263B 2665 2666 2663 2660 2022 25D8 25CB 25D9 2642 2640 266A 266B 263C
107 25BA 25C4 2195 203C 00B6 00A7 25AC 21A8 2191 2193 2192 2190 221F 2194 25B2 25BC
112 my ($table, $offset) = @_;
113 my $def = $table->{$offset} or return;
114 my ($len, $class, $name, $title) = @{$def};
118 $name //= $len <= 2 ? 'res' : 'reserved';
120 if (my $part = $offset/$nibsize % 16) {
122 my $cols = 16 - $part; # remaining
123 $cols = $len if $len < $cols; #TODO: optimise
125 # continued on new row
126 $table->{$offset + $nibsize*$cols} //= [$len*$nibsize, "$class joinu", $name, $title];
132 elsif (my $rows = $len >> 4) {
134 if ($len -= $rows << 4) {
135 # partial row remains
136 $table->{$offset + $nibsize*$rows * 16} //= [$len*$nibsize, "$class joinu", '', $title];
139 $attr .= sprintf ' rowspan=%d', $rows;
143 $attr .= sprintf ' colspan=%d', $len unless $len == 1;
144 $attr .= $1 if $class and $class =~ s/( \w+="[^"]*")//;
145 $attr .= sprintf ' class="%s"', $class if $class;
146 $attr .= sprintf ' title="%s"', EscapeHTML($title) if $title;
147 return "<td$attr>$name";
150 for my $row (@request) {
151 printf '<div class="section"><table class="glyphs%s">', !$row->{cell} && ' charmap';
152 my $title = $row->{set};
153 $title .= " <aside>($_)</aside>" for $row->{setnote} // ();
154 printf '<caption>%s</caption>', $title;
156 for my $section (qw{thead}) {
157 print "<$section><tr><th>↱";
158 print '<th>', $_ for @nibble;
162 for my $msb (0 .. (length($row->{table}) || 256) - 1 >> 4) {
163 printf '<tr><th>%X', ($msb + ($row->{offset} >> 4)) * $nibsize;
164 for my $lsb (0 .. $#nibble) {
165 my $val = ( ($msb<<4) + $lsb ) * $nibsize;
167 if (ref $row->{cell} eq 'CODE') {
168 print $row->{cell}->($val);
171 print range_cell($row->{cell}, $val);
175 my $glyph = substr $row->{table}, $val, 1;
176 if ($glyph eq $NOCHAR) {
181 print "\n".$glyphs->glyph_cell($glyph);
185 print "</table></div>\n";
192 <table class="glyphs"><tr>
193 <td class="X Cc">control
194 <td class="X Zs"><span>whitespace</span>
195 <td class="X Mn">diacritic<table class="glyphs"><tr>
196 <td class="X Sk">letter
198 <td class="X Po">punctuation<table class="glyphs"><tr>
199 <td class="X Pf">quote
201 <td class="X So">symbol<table class="glyphs"><tr>
202 <td class="X Sm">math
203 <td class="X Sc">currency
205 <td class="X No">numeric
206 <td class="X Greek">greek<table class="glyphs"><tr>
207 <td class="X Latin">latin
208 <td class="X Cyrillic">cyrillic
210 <td class="X Aramaic">aramaic<table class="glyphs"><tr>
211 <td class="X Brahmic">brahmic
212 <td class="X Arabic">arabic
214 <td class="X Syllabic">syllabic<table class="glyphs"><tr>
215 <td class="X African">african
216 <td class="X Hiragana">japanese
217 <td class="X Han">cjk
218 <td class="X Bopomofo">chinese
220 <td class="X Alpha">alphabetic
223 <table class="glyphs"><tr>
224 <td class="X">unicode 7.0
225 <td class="X Xr">proposed
226 <td class="X Xd">deprecated
227 <td class="">unassigned
228 <td class="X Xi">invalid