1/* Convert using charmaps and possibly iconv().
2 Copyright (C) 2001-2021 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Ulrich Drepper <drepper@redhat.com>, 2001.
5
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published
8 by the Free Software Foundation; version 2 of the License, or
9 (at your option) any later version.
10
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, see <https://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <error.h>
22#include <fcntl.h>
23#include <iconv.h>
24#include <libintl.h>
25#include <stdio.h>
26#include <stdlib.h>
27#include <unistd.h>
28#include <stdint.h>
29#include <sys/mman.h>
30#include <sys/stat.h>
31
32#include "iconv_prog.h"
33
34
35/* Prototypes for a few program-wide used functions. */
36#include <programs/xmalloc.h>
37
38
39struct convtable
40{
41 int term[256 / 8];
42 union
43 {
44 struct convtable *sub;
45 struct charseq *out;
46 } val[256];
47};
48
49
50static inline struct convtable *
51allocate_table (void)
52{
53 return (struct convtable *) xcalloc (1, sizeof (struct convtable));
54}
55
56static inline void
57free_table (struct convtable *tbl)
58{
59 free (tbl);
60}
61
62
63static inline int
64is_term (struct convtable *tbl, unsigned int idx)
65{
66 return tbl->term[idx / 8] & (1 << (idx % 8));
67}
68
69
70static inline void
71clear_term (struct convtable *tbl, unsigned int idx)
72{
73 tbl->term[idx / 8] &= ~(1 << (idx % 8));
74}
75
76
77static inline void
78set_term (struct convtable *tbl, unsigned int idx)
79{
80 tbl->term[idx / 8] |= 1 << (idx % 8);
81}
82
83
84/* Generate the conversion table. */
85static struct convtable *use_from_charmap (struct charmap_t *from_charmap,
86 const char *to_code);
87static struct convtable *use_to_charmap (const char *from_code,
88 struct charmap_t *to_charmap);
89static struct convtable *use_both_charmaps (struct charmap_t *from_charmap,
90 struct charmap_t *to_charmap);
91
92/* Prototypes for the functions doing the actual work. */
93static int process_block (struct convtable *tbl, char *addr, size_t len,
94 FILE *output);
95static int process_fd (struct convtable *tbl, int fd, FILE *output);
96static int process_file (struct convtable *tbl, FILE *input, FILE *output);
97
98
99int
100charmap_conversion (const char *from_code, struct charmap_t *from_charmap,
101 const char *to_code, struct charmap_t *to_charmap,
102 int argc, int remaining, char *argv[],
103 const char *output_file)
104{
105 struct convtable *cvtbl;
106 int status = EXIT_SUCCESS;
107
108 /* We have three different cases to handle:
109
110 - both, from_charmap and to_charmap, are available. This means we
111 can assume that the symbolic names match and use them to create
112 the mapping.
113
114 - only from_charmap is available. In this case we can only hope that
115 the symbolic names used are of the <Uxxxx> form in which case we
116 can use a UCS4->"to_code" iconv() conversion for the second step.
117
118 - only to_charmap is available. This is similar, only that we would
119 use iconv() for the "to_code"->UCS4 conversion.
120
121 We first create a table which maps input bytes into output bytes.
122 Once this is done we can handle all three of the cases above
123 equally. */
124 if (from_charmap != NULL)
125 {
126 if (to_charmap == NULL)
127 cvtbl = use_from_charmap (from_charmap, to_code);
128 else
129 cvtbl = use_both_charmaps (from_charmap, to_charmap);
130 }
131 else
132 {
133 assert (to_charmap != NULL);
134 cvtbl = use_to_charmap (from_code, to_charmap);
135 }
136
137 /* If we couldn't generate a table stop now. */
138 if (cvtbl == NULL)
139 return EXIT_FAILURE;
140
141 /* Determine output file. */
142 FILE *output;
143 if (output_file != NULL && strcmp (output_file, "-") != 0)
144 {
145 output = fopen (output_file, "w");
146 if (output == NULL)
147 error (EXIT_FAILURE, errno, _("cannot open output file"));
148 }
149 else
150 output = stdout;
151
152 /* We can now start the conversion. */
153 if (remaining == argc)
154 {
155 if (process_file (cvtbl, stdin, output) != 0)
156 status = EXIT_FAILURE;
157 }
158 else
159 do
160 {
161 int fd;
162
163 if (verbose)
164 printf ("%s:\n", argv[remaining]);
165 if (strcmp (argv[remaining], "-") == 0)
166 fd = 0;
167 else
168 {
169 fd = open (argv[remaining], O_RDONLY);
170
171 if (fd == -1)
172 {
173 error (0, errno, _("cannot open input file `%s'"),
174 argv[remaining]);
175 status = EXIT_FAILURE;
176 continue;
177 }
178 }
179
180#ifdef _POSIX_MAPPED_FILES
181 struct stat64 st;
182 char *addr;
183 /* We have possibilities for reading the input file. First try
184 to mmap() it since this will provide the fastest solution. */
185 if (fstat64 (fd, &st) == 0
186 && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
187 fd, 0)) != MAP_FAILED))
188 {
189 /* Yes, we can use mmap(). The descriptor is not needed
190 anymore. */
191 if (close (fd) != 0)
192 error (EXIT_FAILURE, errno,
193 _("error while closing input `%s'"), argv[remaining]);
194
195 if (process_block (cvtbl, addr, st.st_size, output) < 0)
196 {
197 /* Something went wrong. */
198 status = EXIT_FAILURE;
199
200 /* We don't need the input data anymore. */
201 munmap ((void *) addr, st.st_size);
202
203 /* We cannot go on with producing output since it might
204 lead to problem because the last output might leave
205 the output stream in an undefined state. */
206 break;
207 }
208
209 /* We don't need the input data anymore. */
210 munmap ((void *) addr, st.st_size);
211 }
212 else
213#endif /* _POSIX_MAPPED_FILES */
214 {
215 /* Read the file in pieces. */
216 if (process_fd (cvtbl, fd, output) != 0)
217 {
218 /* Something went wrong. */
219 status = EXIT_FAILURE;
220
221 /* We don't need the input file anymore. */
222 close (fd);
223
224 /* We cannot go on with producing output since it might
225 lead to problem because the last output might leave
226 the output stream in an undefined state. */
227 break;
228 }
229
230 /* Now close the file. */
231 close (fd);
232 }
233 }
234 while (++remaining < argc);
235
236 /* All done. */
237 free_table (cvtbl);
238 return status;
239}
240
241
242/* Add the IN->OUT mapping to TBL. OUT is potentially stored in the table.
243 IN is used only here, so it need not be kept live afterwards. */
244static void
245add_bytes (struct convtable *tbl, const struct charseq *in, struct charseq *out)
246{
247 int n = 0;
248 unsigned int byte;
249
250 assert (in->nbytes > 0);
251
252 byte = ((unsigned char *) in->bytes)[n];
253 while (n + 1 < in->nbytes)
254 {
255 if (is_term (tbl, byte) || tbl->val[byte].sub == NULL)
256 {
257 /* Note that we simply ignore a definition for a byte sequence
258 which is also the prefix for a longer one. */
259 clear_term (tbl, byte);
260 tbl->val[byte].sub =
261 (struct convtable *) xcalloc (1, sizeof (struct convtable));
262 }
263
264 tbl = tbl->val[byte].sub;
265
266 byte = ((unsigned char *) in->bytes)[++n];
267 }
268
269 /* Only add the new sequence if there is none yet and the byte sequence
270 is not part of an even longer one. */
271 if (! is_term (tbl, byte) && tbl->val[byte].sub == NULL)
272 {
273 set_term (tbl, byte);
274 tbl->val[byte].out = out;
275 }
276}
277
278/* Try to convert SEQ from WCHAR_T format using CD.
279 Returns a malloc'd struct or NULL. */
280static struct charseq *
281convert_charseq (iconv_t cd, const struct charseq *seq)
282{
283 struct charseq *result = NULL;
284
285 if (seq->ucs4 != UNINITIALIZED_CHAR_VALUE)
286 {
287 /* There is a chance. Try the iconv module. */
288 wchar_t inbuf[1] = { seq->ucs4 };
289 unsigned char outbuf[64];
290 char *inptr = (char *) inbuf;
291 size_t inlen = sizeof (inbuf);
292 char *outptr = (char *) outbuf;
293 size_t outlen = sizeof (outbuf);
294
295 (void) iconv (cd, &inptr, &inlen, &outptr, &outlen);
296
297 if (outptr != (char *) outbuf)
298 {
299 /* We got some output. Good, use it. */
300 outlen = sizeof (outbuf) - outlen;
301 assert ((char *) outbuf + outlen == outptr);
302
303 result = xmalloc (sizeof (struct charseq) + outlen);
304 result->name = seq->name;
305 result->ucs4 = seq->ucs4;
306 result->nbytes = outlen;
307 memcpy (result->bytes, outbuf, outlen);
308 }
309
310 /* Clear any possible state left behind. */
311 (void) iconv (cd, NULL, NULL, NULL, NULL);
312 }
313
314 return result;
315}
316
317
318static struct convtable *
319use_from_charmap (struct charmap_t *from_charmap, const char *to_code)
320{
321 /* We iterate over all entries in the from_charmap and for those which
322 have a known UCS4 representation we use an iconv() call to determine
323 the mapping to the to_code charset. */
324 struct convtable *rettbl;
325 iconv_t cd;
326 void *ptr = NULL;
327 const void *key;
328 size_t keylen;
329 void *data;
330
331 cd = iconv_open (to_code, "WCHAR_T");
332 if (cd == (iconv_t) -1)
333 /* We cannot do anything. */
334 return NULL;
335
336 rettbl = allocate_table ();
337
338 while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
339 >= 0)
340 {
341 struct charseq *in = data;
342 struct charseq *newp = convert_charseq (cd, in);
343 if (newp != NULL)
344 add_bytes (rettbl, in, newp);
345 }
346
347 iconv_close (cd);
348
349 return rettbl;
350}
351
352
353static struct convtable *
354use_to_charmap (const char *from_code, struct charmap_t *to_charmap)
355{
356 /* We iterate over all entries in the to_charmap and for those which
357 have a known UCS4 representation we use an iconv() call to determine
358 the mapping to the from_code charset. */
359 struct convtable *rettbl;
360 iconv_t cd;
361 void *ptr = NULL;
362 const void *key;
363 size_t keylen;
364 void *data;
365
366 /* Note that the conversion we use here is the reverse direction. Without
367 exhaustive search we cannot figure out which input yields the UCS4
368 character we are looking for. Therefore we determine it the other
369 way round. */
370 cd = iconv_open (from_code, "WCHAR_T");
371 if (cd == (iconv_t) -1)
372 /* We cannot do anything. */
373 return NULL;
374
375 rettbl = allocate_table ();
376
377 while (iterate_table (&to_charmap->char_table, &ptr, &key, &keylen, &data)
378 >= 0)
379 {
380 struct charseq *out = data;
381 struct charseq *newp = convert_charseq (cd, out);
382 if (newp != NULL)
383 {
384 add_bytes (rettbl, newp, out);
385 free (newp);
386 }
387 }
388
389 iconv_close (cd);
390
391 return rettbl;
392}
393
394
395static struct convtable *
396use_both_charmaps (struct charmap_t *from_charmap,
397 struct charmap_t *to_charmap)
398{
399 /* In this case we iterate over all the entries in the from_charmap,
400 determine the internal name, and find an appropriate entry in the
401 to_charmap (if it exists). */
402 struct convtable *rettbl = allocate_table ();
403 void *ptr = NULL;
404 const void *key;
405 size_t keylen;
406 void *data;
407
408 while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
409 >= 0)
410 {
411 struct charseq *in = (struct charseq *) data;
412 struct charseq *out = charmap_find_value (to_charmap, key, keylen);
413
414 if (out != NULL)
415 add_bytes (rettbl, in, out);
416 }
417
418 return rettbl;
419}
420
421
422static int
423process_block (struct convtable *tbl, char *addr, size_t len, FILE *output)
424{
425 size_t n = 0;
426
427 while (n < len)
428 {
429 struct convtable *cur = tbl;
430 unsigned char *curp = (unsigned char *) addr;
431 unsigned int byte = *curp;
432 int cnt;
433 struct charseq *out;
434
435 while (! is_term (cur, byte))
436 if (cur->val[byte].sub == NULL)
437 {
438 /* This is an invalid sequence. Skip the first byte if we are
439 ignoring errors. Otherwise punt. */
440 if (! omit_invalid)
441 {
442 error (0, 0, _("illegal input sequence at position %Zd"), n);
443 return -1;
444 }
445
446 n -= curp - (unsigned char *) addr;
447
448 byte = *(curp = (unsigned char *) ++addr);
449 if (++n >= len)
450 /* All converted. */
451 return 0;
452
453 cur = tbl;
454 }
455 else
456 {
457 cur = cur->val[byte].sub;
458
459 if (++n >= len)
460 {
461 error (0, 0, _("\
462incomplete character or shift sequence at end of buffer"));
463 return -1;
464 }
465
466 byte = *++curp;
467 }
468
469 /* We found a final byte. Write the output bytes. */
470 out = cur->val[byte].out;
471 for (cnt = 0; cnt < out->nbytes; ++cnt)
472 fputc_unlocked (out->bytes[cnt], output);
473
474 addr = (char *) curp + 1;
475 ++n;
476 }
477
478 return 0;
479}
480
481
482static int
483process_fd (struct convtable *tbl, int fd, FILE *output)
484{
485 /* We have a problem with reading from a descriptor since we must not
486 provide the iconv() function an incomplete character or shift
487 sequence at the end of the buffer. Since we have to deal with
488 arbitrary encodings we must read the whole text in a buffer and
489 process it in one step. */
490 static char *inbuf = NULL;
491 static size_t maxlen = 0;
492 char *inptr = inbuf;
493 size_t actlen = 0;
494
495 while (actlen < maxlen)
496 {
497 ssize_t n = read (fd, inptr, maxlen - actlen);
498
499 if (n == 0)
500 /* No more text to read. */
501 break;
502
503 if (n == -1)
504 {
505 /* Error while reading. */
506 error (0, errno, _("error while reading the input"));
507 return -1;
508 }
509
510 inptr += n;
511 actlen += n;
512 }
513
514 if (actlen == maxlen)
515 while (1)
516 {
517 ssize_t n;
518 char *new_inbuf;
519
520 /* Increase the buffer. */
521 new_inbuf = (char *) realloc (inbuf, maxlen + 32768);
522 if (new_inbuf == NULL)
523 {
524 error (0, errno, _("unable to allocate buffer for input"));
525 return -1;
526 }
527 inbuf = new_inbuf;
528 maxlen += 32768;
529 inptr = inbuf + actlen;
530
531 do
532 {
533 n = read (fd, inptr, maxlen - actlen);
534
535 if (n == 0)
536 /* No more text to read. */
537 break;
538
539 if (n == -1)
540 {
541 /* Error while reading. */
542 error (0, errno, _("error while reading the input"));
543 return -1;
544 }
545
546 inptr += n;
547 actlen += n;
548 }
549 while (actlen < maxlen);
550
551 if (n == 0)
552 /* Break again so we leave both loops. */
553 break;
554 }
555
556 /* Now we have all the input in the buffer. Process it in one run. */
557 return process_block (tbl, inbuf, actlen, output);
558}
559
560
561static int
562process_file (struct convtable *tbl, FILE *input, FILE *output)
563{
564 /* This should be safe since we use this function only for `stdin' and
565 we haven't read anything so far. */
566 return process_fd (tbl, fileno (input), output);
567}
568