2 * Copyright (c) 2009-2011 Petri Lehtinen <petri@digip.org>
4 * Jansson is free software; you can redistribute it and/or modify
5 * it under the terms of the MIT license. See LICENSE for details.
19 #include "jansson_private.h"
20 #include "strbuffer.h"
23 #define STREAM_STATE_OK 0
24 #define STREAM_STATE_EOF -1
25 #define STREAM_STATE_ERROR -2
27 #define TOKEN_INVALID -1
29 #define TOKEN_STRING 256
30 #define TOKEN_INTEGER 257
31 #define TOKEN_REAL 258
32 #define TOKEN_TRUE 259
33 #define TOKEN_FALSE 260
34 #define TOKEN_NULL 261
36 /* Read one byte from stream, convert to unsigned char, then int, and
37 return. return EOF on end of file. This corresponds to the
38 behaviour of fgetc(). */
39 typedef int (*get_func)(void *data);
48 int column, last_column;
54 strbuffer_t saved_text;
63 #define stream_to_lex(stream) container_of(stream, lex_t, stream)
66 /*** error reporting ***/
68 static void error_set(json_error_t *error, const lex_t *lex,
72 char msg_text[JSON_ERROR_TEXT_LENGTH];
74 int line = -1, col = -1;
76 const char *result = msg_text;
82 vsnprintf(msg_text, JSON_ERROR_TEXT_LENGTH, msg, ap);
87 const char *saved_text = strbuffer_value(&lex->saved_text);
88 char msg_with_context[JSON_ERROR_TEXT_LENGTH];
90 line = lex->stream.line;
91 col = lex->stream.column;
92 pos = lex->stream.position;
94 if(saved_text && saved_text[0])
96 if(lex->saved_text.length <= 20) {
97 snprintf(msg_with_context, JSON_ERROR_TEXT_LENGTH,
98 "%s near '%s'", msg_text, saved_text);
99 result = msg_with_context;
104 if(lex->stream.state == STREAM_STATE_ERROR) {
105 /* No context for UTF-8 decoding errors */
109 snprintf(msg_with_context, JSON_ERROR_TEXT_LENGTH,
110 "%s near end of file", msg_text);
111 result = msg_with_context;
116 jsonp_error_set(error, line, col, pos, "%s", result);
120 /*** lexical analyzer ***/
123 stream_init(stream_t *stream, get_func get, void *data)
127 stream->buffer[0] = '\0';
128 stream->buffer_pos = 0;
130 stream->state = STREAM_STATE_OK;
133 stream->position = 0;
136 static int stream_get(stream_t *stream, json_error_t *error)
140 if(stream->state != STREAM_STATE_OK)
141 return stream->state;
143 if(!stream->buffer[stream->buffer_pos])
145 c = stream->get(stream->data);
147 stream->state = STREAM_STATE_EOF;
148 return STREAM_STATE_EOF;
151 stream->buffer[0] = c;
152 stream->buffer_pos = 0;
154 if(0x80 <= c && c <= 0xFF)
156 /* multi-byte UTF-8 sequence */
159 count = utf8_check_first(c);
165 for(i = 1; i < count; i++)
166 stream->buffer[i] = stream->get(stream->data);
168 if(!utf8_check_full(stream->buffer, count, NULL))
171 stream->buffer[count] = '\0';
174 stream->buffer[1] = '\0';
177 c = stream->buffer[stream->buffer_pos++];
182 stream->last_column = stream->column;
185 else if(utf8_check_first(c)) {
186 /* track the Unicode character column, so increment only if
187 this is the first character of a UTF-8 sequence */
194 stream->state = STREAM_STATE_ERROR;
195 error_set(error, stream_to_lex(stream), "unable to decode byte 0x%x", c);
196 return STREAM_STATE_ERROR;
199 static void stream_unget(stream_t *stream, int c)
201 if(c == STREAM_STATE_EOF || c == STREAM_STATE_ERROR)
207 stream->column = stream->last_column;
209 else if(utf8_check_first(c))
212 assert(stream->buffer_pos > 0);
213 stream->buffer_pos--;
214 assert(stream->buffer[stream->buffer_pos] == c);
218 static int lex_get(lex_t *lex, json_error_t *error)
220 return stream_get(&lex->stream, error);
223 static void lex_save(lex_t *lex, int c)
225 strbuffer_append_byte(&lex->saved_text, c);
228 static int lex_get_save(lex_t *lex, json_error_t *error)
230 int c = stream_get(&lex->stream, error);
231 if(c != STREAM_STATE_EOF && c != STREAM_STATE_ERROR)
236 static void lex_unget(lex_t *lex, int c)
238 stream_unget(&lex->stream, c);
241 static void lex_unget_unsave(lex_t *lex, int c)
243 if(c != STREAM_STATE_EOF && c != STREAM_STATE_ERROR) {
245 stream_unget(&lex->stream, c);
246 d = strbuffer_pop(&lex->saved_text);
251 static void lex_save_cached(lex_t *lex)
253 while(lex->stream.buffer[lex->stream.buffer_pos] != '\0')
255 lex_save(lex, lex->stream.buffer[lex->stream.buffer_pos]);
256 lex->stream.buffer_pos++;
257 lex->stream.position++;
261 /* assumes that str points to 'u' plus at least 4 valid hex digits */
262 static int32_t decode_unicode_escape(const char *str)
267 assert(str[0] == 'u');
269 for(i = 1; i <= 4; i++) {
275 value += c - 'a' + 10;
277 value += c - 'A' + 10;
285 static void lex_scan_string(lex_t *lex, json_error_t *error)
292 lex->value.string = NULL;
293 lex->token = TOKEN_INVALID;
295 c = lex_get_save(lex, error);
298 if(c == STREAM_STATE_ERROR)
301 else if(c == STREAM_STATE_EOF) {
302 error_set(error, lex, "premature end of input");
306 else if(0 <= c && c <= 0x1F) {
307 /* control character */
308 lex_unget_unsave(lex, c);
310 error_set(error, lex, "unexpected newline", c);
312 error_set(error, lex, "control character 0x%x", c);
317 c = lex_get_save(lex, error);
319 c = lex_get_save(lex, error);
320 for(i = 0; i < 4; i++) {
322 error_set(error, lex, "invalid escape");
325 c = lex_get_save(lex, error);
328 else if(c == '"' || c == '\\' || c == '/' || c == 'b' ||
329 c == 'f' || c == 'n' || c == 'r' || c == 't')
330 c = lex_get_save(lex, error);
332 error_set(error, lex, "invalid escape");
337 c = lex_get_save(lex, error);
340 /* the actual value is at most of the same length as the source
342 - shortcut escapes (e.g. "\t") (length 2) are converted to 1 byte
343 - a single \uXXXX escape (length 6) is converted to at most 3 bytes
344 - two \uXXXX escapes (length 12) forming an UTF-16 surrogate pair
345 are converted to 4 bytes
347 lex->value.string = jsonp_malloc(lex->saved_text.length + 1);
348 if(!lex->value.string) {
349 /* this is not very nice, since TOKEN_INVALID is returned */
354 t = lex->value.string;
356 /* + 1 to skip the " */
357 p = strbuffer_value(&lex->saved_text) + 1;
367 value = decode_unicode_escape(p);
370 if(0xD800 <= value && value <= 0xDBFF) {
372 if(*p == '\\' && *(p + 1) == 'u') {
373 int32_t value2 = decode_unicode_escape(++p);
376 if(0xDC00 <= value2 && value2 <= 0xDFFF) {
377 /* valid second surrogate */
379 ((value - 0xD800) << 10) +
384 /* invalid second surrogate */
385 error_set(error, lex,
386 "invalid Unicode '\\u%04X\\u%04X'",
392 /* no second surrogate */
393 error_set(error, lex, "invalid Unicode '\\u%04X'",
398 else if(0xDC00 <= value && value <= 0xDFFF) {
399 error_set(error, lex, "invalid Unicode '\\u%04X'", value);
404 error_set(error, lex, "\\u0000 is not allowed");
408 if(utf8_encode(value, buffer, &length))
411 memcpy(t, buffer, length);
416 case '"': case '\\': case '/':
418 case 'b': *t = '\b'; break;
419 case 'f': *t = '\f'; break;
420 case 'n': *t = '\n'; break;
421 case 'r': *t = '\r'; break;
422 case 't': *t = '\t'; break;
433 lex->token = TOKEN_STRING;
437 jsonp_free(lex->value.string);
440 #if JSON_INTEGER_IS_LONG_LONG
441 #define json_strtoint strtoll
443 #define json_strtoint strtol
446 static int lex_scan_number(lex_t *lex, int c, json_error_t *error)
448 const char *saved_text;
452 lex->token = TOKEN_INVALID;
455 c = lex_get_save(lex, error);
458 c = lex_get_save(lex, error);
460 lex_unget_unsave(lex, c);
464 else if(isdigit(c)) {
465 c = lex_get_save(lex, error);
467 c = lex_get_save(lex, error);
470 lex_unget_unsave(lex, c);
474 if(c != '.' && c != 'E' && c != 'e') {
477 lex_unget_unsave(lex, c);
479 saved_text = strbuffer_value(&lex->saved_text);
482 value = json_strtoint(saved_text, &end, 10);
483 if(errno == ERANGE) {
485 error_set(error, lex, "too big negative integer");
487 error_set(error, lex, "too big integer");
491 assert(end == saved_text + lex->saved_text.length);
493 lex->token = TOKEN_INTEGER;
494 lex->value.integer = value;
499 c = lex_get(lex, error);
506 c = lex_get_save(lex, error);
508 c = lex_get_save(lex, error);
511 if(c == 'E' || c == 'e') {
512 c = lex_get_save(lex, error);
513 if(c == '+' || c == '-')
514 c = lex_get_save(lex, error);
517 lex_unget_unsave(lex, c);
521 c = lex_get_save(lex, error);
523 c = lex_get_save(lex, error);
526 lex_unget_unsave(lex, c);
528 saved_text = strbuffer_value(&lex->saved_text);
529 value = strtod(saved_text, &end);
530 assert(end == saved_text + lex->saved_text.length);
532 if(errno == ERANGE && value != 0) {
533 error_set(error, lex, "real number overflow");
537 lex->token = TOKEN_REAL;
538 lex->value.real = value;
545 static int lex_scan(lex_t *lex, json_error_t *error)
549 strbuffer_clear(&lex->saved_text);
551 if(lex->token == TOKEN_STRING) {
552 jsonp_free(lex->value.string);
553 lex->value.string = NULL;
556 c = lex_get(lex, error);
557 while(c == ' ' || c == '\t' || c == '\n' || c == '\r')
558 c = lex_get(lex, error);
560 if(c == STREAM_STATE_EOF) {
561 lex->token = TOKEN_EOF;
565 if(c == STREAM_STATE_ERROR) {
566 lex->token = TOKEN_INVALID;
572 if(c == '{' || c == '}' || c == '[' || c == ']' || c == ':' || c == ',')
576 lex_scan_string(lex, error);
578 else if(isdigit(c) || c == '-') {
579 if(lex_scan_number(lex, c, error))
583 else if(isupper(c) || islower(c)) {
584 /* eat up the whole identifier for clearer error messages */
585 const char *saved_text;
587 c = lex_get_save(lex, error);
588 while(isupper(c) || islower(c))
589 c = lex_get_save(lex, error);
590 lex_unget_unsave(lex, c);
592 saved_text = strbuffer_value(&lex->saved_text);
594 if(strcmp(saved_text, "true") == 0)
595 lex->token = TOKEN_TRUE;
596 else if(strcmp(saved_text, "false") == 0)
597 lex->token = TOKEN_FALSE;
598 else if(strcmp(saved_text, "null") == 0)
599 lex->token = TOKEN_NULL;
601 lex->token = TOKEN_INVALID;
605 /* save the rest of the input UTF-8 sequence to get an error
606 message of valid UTF-8 */
607 lex_save_cached(lex);
608 lex->token = TOKEN_INVALID;
615 static char *lex_steal_string(lex_t *lex)
618 if(lex->token == TOKEN_STRING)
620 result = lex->value.string;
621 lex->value.string = NULL;
626 static int lex_init(lex_t *lex, get_func get, void *data)
628 stream_init(&lex->stream, get, data);
629 if(strbuffer_init(&lex->saved_text))
632 lex->token = TOKEN_INVALID;
636 static void lex_close(lex_t *lex)
638 if(lex->token == TOKEN_STRING)
639 jsonp_free(lex->value.string);
640 strbuffer_close(&lex->saved_text);
646 static json_t *parse_value(lex_t *lex, json_error_t *error);
648 static json_t *parse_object(lex_t *lex, json_error_t *error)
650 json_t *object = json_object();
654 lex_scan(lex, error);
655 if(lex->token == '}')
662 if(lex->token != TOKEN_STRING) {
663 error_set(error, lex, "string or '}' expected");
667 key = lex_steal_string(lex);
671 lex_scan(lex, error);
672 if(lex->token != ':') {
674 error_set(error, lex, "':' expected");
678 lex_scan(lex, error);
679 value = parse_value(lex, error);
685 if(json_object_set_nocheck(object, key, value)) {
694 lex_scan(lex, error);
695 if(lex->token != ',')
698 lex_scan(lex, error);
701 if(lex->token != '}') {
702 error_set(error, lex, "'}' expected");
713 static json_t *parse_array(lex_t *lex, json_error_t *error)
715 json_t *array = json_array();
719 lex_scan(lex, error);
720 if(lex->token == ']')
724 json_t *elem = parse_value(lex, error);
728 if(json_array_append(array, elem)) {
734 lex_scan(lex, error);
735 if(lex->token != ',')
738 lex_scan(lex, error);
741 if(lex->token != ']') {
742 error_set(error, lex, "']' expected");
753 static json_t *parse_value(lex_t *lex, json_error_t *error)
759 json = json_string_nocheck(lex->value.string);
763 case TOKEN_INTEGER: {
764 json = json_integer(lex->value.integer);
769 json = json_real(lex->value.real);
786 json = parse_object(lex, error);
790 json = parse_array(lex, error);
794 error_set(error, lex, "invalid token");
798 error_set(error, lex, "unexpected token");
808 static json_t *parse_json(lex_t *lex, json_error_t *error)
810 lex_scan(lex, error);
811 if(lex->token != '[' && lex->token != '{') {
812 error_set(error, lex, "'[' or '{' expected");
816 return parse_value(lex, error);
825 static int string_get(void *data)
828 string_data_t *stream = (string_data_t *)data;
829 c = stream->data[stream->pos];
835 return (unsigned char)c;
839 json_t *json_loads(const char *string, size_t flags, json_error_t *error)
843 string_data_t stream_data = {string, 0};
845 (void)flags; /* unused */
847 if(lex_init(&lex, string_get, (void *)&stream_data))
850 jsonp_error_init(error, "<string>");
852 result = parse_json(&lex, error);
856 lex_scan(&lex, error);
857 if(lex.token != TOKEN_EOF) {
858 error_set(error, &lex, "end of file expected");
868 json_t *json_loadf(FILE *input, size_t flags, json_error_t *error)
873 (void)flags; /* unused */
875 if(lex_init(&lex, (get_func)fgetc, input))
883 jsonp_error_init(error, source);
885 result = parse_json(&lex, error);
889 lex_scan(&lex, error);
890 if(lex.token != TOKEN_EOF) {
891 error_set(error, &lex, "end of file expected");
901 json_t *json_load_file(const char *path, size_t flags, json_error_t *error)
906 jsonp_error_init(error, path);
908 fp = fopen(path, "r");
911 error_set(error, NULL, "unable to open %s: %s",
912 path, strerror(errno));
916 result = json_loadf(fp, flags, error);