2 * Copyright (c) 2009, 2010 Petri Lehtinen <petri@digip.org>
4 * Jansson is free software; you can redistribute it and/or modify
5 * it under the terms of the MIT license. See LICENSE for details.
19 #include "jansson_private.h"
20 #include "strbuffer.h"
23 #define TOKEN_INVALID -1
25 #define TOKEN_STRING 256
26 #define TOKEN_INTEGER 257
27 #define TOKEN_REAL 258
28 #define TOKEN_TRUE 259
29 #define TOKEN_FALSE 260
30 #define TOKEN_NULL 261
32 /* read one byte from stream, return EOF on end of file */
33 typedef int (*get_func)(void *data);
35 /* return non-zero if end of file has been reached */
36 typedef int (*eof_func)(void *data);
50 strbuffer_t saved_text;
61 /*** error reporting ***/
63 static void error_set(json_error_t *error, const lex_t *lex,
67 char msg_text[JSON_ERROR_TEXT_LENGTH];
69 int line = -1, col = -1;
70 const char *result = msg_text;
76 vsnprintf(msg_text, JSON_ERROR_TEXT_LENGTH, msg, ap);
81 const char *saved_text = strbuffer_value(&lex->saved_text);
82 char msg_with_context[JSON_ERROR_TEXT_LENGTH];
86 if(saved_text && saved_text[0])
88 if(lex->saved_text.length <= 20) {
89 snprintf(msg_with_context, JSON_ERROR_TEXT_LENGTH,
90 "%s near '%s'", msg_text, saved_text);
91 result = msg_with_context;
96 snprintf(msg_with_context, JSON_ERROR_TEXT_LENGTH,
97 "%s near end of file", msg_text);
98 result = msg_with_context;
102 jsonp_error_set(error, line, col, "%s", result);
106 /*** lexical analyzer ***/
109 stream_init(stream_t *stream, get_func get, eof_func eof, void *data)
114 stream->stream_pos = 0;
115 stream->buffer[0] = '\0';
116 stream->buffer_pos = 0;
119 static char stream_get(stream_t *stream, json_error_t *error)
123 if(!stream->buffer[stream->buffer_pos])
125 stream->buffer[0] = stream->get(stream->data);
126 stream->buffer_pos = 0;
128 c = stream->buffer[0];
130 if((unsigned char)c >= 0x80 && c != (char)EOF)
132 /* multi-byte UTF-8 sequence */
135 count = utf8_check_first(c);
141 for(i = 1; i < count; i++)
142 stream->buffer[i] = stream->get(stream->data);
144 if(!utf8_check_full(stream->buffer, count, NULL))
147 stream->stream_pos += count;
148 stream->buffer[count] = '\0';
151 stream->buffer[1] = '\0';
152 stream->stream_pos++;
156 return stream->buffer[stream->buffer_pos++];
159 error_set(error, NULL, "unable to decode byte 0x%x at position %d",
160 (unsigned char)c, stream->stream_pos);
162 stream->buffer[0] = EOF;
163 stream->buffer[1] = '\0';
164 stream->buffer_pos = 1;
169 static void stream_unget(stream_t *stream, char c)
171 assert(stream->buffer_pos > 0);
172 stream->buffer_pos--;
173 assert(stream->buffer[stream->buffer_pos] == c);
177 static int lex_get(lex_t *lex, json_error_t *error)
179 return stream_get(&lex->stream, error);
182 static int lex_eof(lex_t *lex)
184 return lex->stream.eof(lex->stream.data);
187 static void lex_save(lex_t *lex, char c)
189 strbuffer_append_byte(&lex->saved_text, c);
192 static int lex_get_save(lex_t *lex, json_error_t *error)
194 char c = stream_get(&lex->stream, error);
199 static void lex_unget_unsave(lex_t *lex, char c)
202 stream_unget(&lex->stream, c);
203 d = strbuffer_pop(&lex->saved_text);
207 static void lex_save_cached(lex_t *lex)
209 while(lex->stream.buffer[lex->stream.buffer_pos] != '\0')
211 lex_save(lex, lex->stream.buffer[lex->stream.buffer_pos]);
212 lex->stream.buffer_pos++;
216 /* assumes that str points to 'u' plus at least 4 valid hex digits */
217 static int32_t decode_unicode_escape(const char *str)
222 assert(str[0] == 'u');
224 for(i = 1; i <= 4; i++) {
230 value += c - 'a' + 10;
232 value += c - 'A' + 10;
240 static void lex_scan_string(lex_t *lex, json_error_t *error)
247 lex->value.string = NULL;
248 lex->token = TOKEN_INVALID;
250 c = lex_get_save(lex, error);
254 lex_unget_unsave(lex, c);
256 error_set(error, lex, "premature end of input");
260 else if((unsigned char)c <= 0x1F) {
261 /* control character */
262 lex_unget_unsave(lex, c);
264 error_set(error, lex, "unexpected newline", c);
266 error_set(error, lex, "control character 0x%x", c);
271 c = lex_get_save(lex, error);
273 c = lex_get_save(lex, error);
274 for(i = 0; i < 4; i++) {
276 lex_unget_unsave(lex, c);
277 error_set(error, lex, "invalid escape");
280 c = lex_get_save(lex, error);
283 else if(c == '"' || c == '\\' || c == '/' || c == 'b' ||
284 c == 'f' || c == 'n' || c == 'r' || c == 't')
285 c = lex_get_save(lex, error);
287 lex_unget_unsave(lex, c);
288 error_set(error, lex, "invalid escape");
293 c = lex_get_save(lex, error);
296 /* the actual value is at most of the same length as the source
298 - shortcut escapes (e.g. "\t") (length 2) are converted to 1 byte
299 - a single \uXXXX escape (length 6) is converted to at most 3 bytes
300 - two \uXXXX escapes (length 12) forming an UTF-16 surrogate pair
301 are converted to 4 bytes
303 lex->value.string = malloc(lex->saved_text.length + 1);
304 if(!lex->value.string) {
305 /* this is not very nice, since TOKEN_INVALID is returned */
310 t = lex->value.string;
312 /* + 1 to skip the " */
313 p = strbuffer_value(&lex->saved_text) + 1;
323 value = decode_unicode_escape(p);
326 if(0xD800 <= value && value <= 0xDBFF) {
328 if(*p == '\\' && *(p + 1) == 'u') {
329 int32_t value2 = decode_unicode_escape(++p);
332 if(0xDC00 <= value2 && value2 <= 0xDFFF) {
333 /* valid second surrogate */
335 ((value - 0xD800) << 10) +
340 /* invalid second surrogate */
341 error_set(error, lex,
342 "invalid Unicode '\\u%04X\\u%04X'",
348 /* no second surrogate */
349 error_set(error, lex, "invalid Unicode '\\u%04X'",
354 else if(0xDC00 <= value && value <= 0xDFFF) {
355 error_set(error, lex, "invalid Unicode '\\u%04X'", value);
360 error_set(error, lex, "\\u0000 is not allowed");
364 if(utf8_encode(value, buffer, &length))
367 memcpy(t, buffer, length);
372 case '"': case '\\': case '/':
374 case 'b': *t = '\b'; break;
375 case 'f': *t = '\f'; break;
376 case 'n': *t = '\n'; break;
377 case 'r': *t = '\r'; break;
378 case 't': *t = '\t'; break;
389 lex->token = TOKEN_STRING;
393 free(lex->value.string);
396 #if JSON_INTEGER_IS_LONG_LONG
397 #define json_strtoint strtoll
399 #define json_strtoint strtol
402 static int lex_scan_number(lex_t *lex, char c, json_error_t *error)
404 const char *saved_text;
408 lex->token = TOKEN_INVALID;
411 c = lex_get_save(lex, error);
414 c = lex_get_save(lex, error);
416 lex_unget_unsave(lex, c);
420 else if(isdigit(c)) {
421 c = lex_get_save(lex, error);
423 c = lex_get_save(lex, error);
426 lex_unget_unsave(lex, c);
430 if(c != '.' && c != 'E' && c != 'e') {
433 lex_unget_unsave(lex, c);
435 saved_text = strbuffer_value(&lex->saved_text);
438 value = json_strtoint(saved_text, &end, 10);
439 if(errno == ERANGE) {
441 error_set(error, lex, "too big negative integer");
443 error_set(error, lex, "too big integer");
447 assert(end == saved_text + lex->saved_text.length);
449 lex->token = TOKEN_INTEGER;
450 lex->value.integer = value;
455 c = lex_get(lex, error);
460 c = lex_get_save(lex, error);
462 c = lex_get_save(lex, error);
465 if(c == 'E' || c == 'e') {
466 c = lex_get_save(lex, error);
467 if(c == '+' || c == '-')
468 c = lex_get_save(lex, error);
471 lex_unget_unsave(lex, c);
475 c = lex_get_save(lex, error);
477 c = lex_get_save(lex, error);
480 lex_unget_unsave(lex, c);
482 saved_text = strbuffer_value(&lex->saved_text);
483 value = strtod(saved_text, &end);
484 assert(end == saved_text + lex->saved_text.length);
486 if(errno == ERANGE && value != 0) {
487 error_set(error, lex, "real number overflow");
491 lex->token = TOKEN_REAL;
492 lex->value.real = value;
499 static int lex_scan(lex_t *lex, json_error_t *error)
503 strbuffer_clear(&lex->saved_text);
505 if(lex->token == TOKEN_STRING) {
506 free(lex->value.string);
507 lex->value.string = NULL;
510 c = lex_get(lex, error);
511 while(c == ' ' || c == '\t' || c == '\n' || c == '\r')
516 c = lex_get(lex, error);
521 lex->token = TOKEN_EOF;
523 lex->token = TOKEN_INVALID;
529 if(c == '{' || c == '}' || c == '[' || c == ']' || c == ':' || c == ',')
533 lex_scan_string(lex, error);
535 else if(isdigit(c) || c == '-') {
536 if(lex_scan_number(lex, c, error))
540 else if(isupper(c) || islower(c)) {
541 /* eat up the whole identifier for clearer error messages */
542 const char *saved_text;
544 c = lex_get_save(lex, error);
545 while(isupper(c) || islower(c))
546 c = lex_get_save(lex, error);
547 lex_unget_unsave(lex, c);
549 saved_text = strbuffer_value(&lex->saved_text);
551 if(strcmp(saved_text, "true") == 0)
552 lex->token = TOKEN_TRUE;
553 else if(strcmp(saved_text, "false") == 0)
554 lex->token = TOKEN_FALSE;
555 else if(strcmp(saved_text, "null") == 0)
556 lex->token = TOKEN_NULL;
558 lex->token = TOKEN_INVALID;
562 /* save the rest of the input UTF-8 sequence to get an error
563 message of valid UTF-8 */
564 lex_save_cached(lex);
565 lex->token = TOKEN_INVALID;
572 static char *lex_steal_string(lex_t *lex)
575 if(lex->token == TOKEN_STRING)
577 result = lex->value.string;
578 lex->value.string = NULL;
583 static int lex_init(lex_t *lex, get_func get, eof_func eof, void *data)
585 stream_init(&lex->stream, get, eof, data);
586 if(strbuffer_init(&lex->saved_text))
589 lex->token = TOKEN_INVALID;
595 static void lex_close(lex_t *lex)
597 if(lex->token == TOKEN_STRING)
598 free(lex->value.string);
599 strbuffer_close(&lex->saved_text);
605 static json_t *parse_value(lex_t *lex, json_error_t *error);
607 static json_t *parse_object(lex_t *lex, json_error_t *error)
609 json_t *object = json_object();
613 lex_scan(lex, error);
614 if(lex->token == '}')
621 if(lex->token != TOKEN_STRING) {
622 error_set(error, lex, "string or '}' expected");
626 key = lex_steal_string(lex);
630 lex_scan(lex, error);
631 if(lex->token != ':') {
633 error_set(error, lex, "':' expected");
637 lex_scan(lex, error);
638 value = parse_value(lex, error);
644 if(json_object_set_nocheck(object, key, value)) {
653 lex_scan(lex, error);
654 if(lex->token != ',')
657 lex_scan(lex, error);
660 if(lex->token != '}') {
661 error_set(error, lex, "'}' expected");
672 static json_t *parse_array(lex_t *lex, json_error_t *error)
674 json_t *array = json_array();
678 lex_scan(lex, error);
679 if(lex->token == ']')
683 json_t *elem = parse_value(lex, error);
687 if(json_array_append(array, elem)) {
693 lex_scan(lex, error);
694 if(lex->token != ',')
697 lex_scan(lex, error);
700 if(lex->token != ']') {
701 error_set(error, lex, "']' expected");
712 static json_t *parse_value(lex_t *lex, json_error_t *error)
718 json = json_string_nocheck(lex->value.string);
722 case TOKEN_INTEGER: {
723 json = json_integer(lex->value.integer);
728 json = json_real(lex->value.real);
745 json = parse_object(lex, error);
749 json = parse_array(lex, error);
753 error_set(error, lex, "invalid token");
757 error_set(error, lex, "unexpected token");
767 static json_t *parse_json(lex_t *lex, json_error_t *error)
769 lex_scan(lex, error);
770 if(lex->token != '[' && lex->token != '{') {
771 error_set(error, lex, "'[' or '{' expected");
775 return parse_value(lex, error);
784 static int string_get(void *data)
787 string_data_t *stream = (string_data_t *)data;
788 c = stream->data[stream->pos];
798 static int string_eof(void *data)
800 string_data_t *stream = (string_data_t *)data;
801 return (stream->data[stream->pos] == '\0');
804 json_t *json_loads(const char *string, size_t flags, json_error_t *error)
808 string_data_t stream_data = {string, 0};
810 (void)flags; /* unused */
812 if(lex_init(&lex, string_get, string_eof, (void *)&stream_data))
815 jsonp_error_init(error, "<string>");
817 result = parse_json(&lex, error);
821 lex_scan(&lex, error);
822 if(lex.token != TOKEN_EOF) {
823 error_set(error, &lex, "end of file expected");
833 json_t *json_loadf(FILE *input, size_t flags, json_error_t *error)
838 (void)flags; /* unused */
840 if(lex_init(&lex, (get_func)fgetc, (eof_func)feof, input))
848 jsonp_error_init(error, source);
850 result = parse_json(&lex, error);
854 lex_scan(&lex, error);
855 if(lex.token != TOKEN_EOF) {
856 error_set(error, &lex, "end of file expected");
866 json_t *json_load_file(const char *path, size_t flags, json_error_t *error)
871 jsonp_error_init(error, path);
873 fp = fopen(path, "r");
876 error_set(error, NULL, "unable to open %s: %s",
877 path, strerror(errno));
881 result = json_loadf(fp, flags, error);