Ruby  2.4.2p198(2017-09-14revision59899)
scanner.c
Go to the documentation of this file.
1 
2 /*
3  * Introduction
4  * ************
5  *
6  * The following notes assume that you are familiar with the YAML specification
7  * (http://yaml.org/spec/cvs/current.html). We mostly follow it, although in
8  * some cases we are less restrictive that it requires.
9  *
10  * The process of transforming a YAML stream into a sequence of events is
11  * divided on two steps: Scanning and Parsing.
12  *
13  * The Scanner transforms the input stream into a sequence of tokens, while the
14  * parser transform the sequence of tokens produced by the Scanner into a
15  * sequence of parsing events.
16  *
17  * The Scanner is rather clever and complicated. The Parser, on the contrary,
18  * is a straightforward implementation of a recursive-descendant parser (or,
19  * LL(1) parser, as it is usually called).
20  *
21  * Actually there are two issues of Scanning that might be called "clever", the
22  * rest is quite straightforward. The issues are "block collection start" and
23  * "simple keys". Both issues are explained below in details.
24  *
25  * Here the Scanning step is explained and implemented. We start with the list
26  * of all the tokens produced by the Scanner together with short descriptions.
27  *
28  * Now, tokens:
29  *
30  * STREAM-START(encoding) # The stream start.
31  * STREAM-END # The stream end.
32  * VERSION-DIRECTIVE(major,minor) # The '%YAML' directive.
33  * TAG-DIRECTIVE(handle,prefix) # The '%TAG' directive.
34  * DOCUMENT-START # '---'
35  * DOCUMENT-END # '...'
36  * BLOCK-SEQUENCE-START # Indentation increase denoting a block
37  * BLOCK-MAPPING-START # sequence or a block mapping.
38  * BLOCK-END # Indentation decrease.
39  * FLOW-SEQUENCE-START # '['
40  * FLOW-SEQUENCE-END # ']'
41  * BLOCK-SEQUENCE-START # '{'
42  * BLOCK-SEQUENCE-END # '}'
43  * BLOCK-ENTRY # '-'
44  * FLOW-ENTRY # ','
45  * KEY # '?' or nothing (simple keys).
46  * VALUE # ':'
47  * ALIAS(anchor) # '*anchor'
48  * ANCHOR(anchor) # '&anchor'
49  * TAG(handle,suffix) # '!handle!suffix'
50  * SCALAR(value,style) # A scalar.
51  *
52  * The following two tokens are "virtual" tokens denoting the beginning and the
53  * end of the stream:
54  *
55  * STREAM-START(encoding)
56  * STREAM-END
57  *
58  * We pass the information about the input stream encoding with the
59  * STREAM-START token.
60  *
61  * The next two tokens are responsible for tags:
62  *
63  * VERSION-DIRECTIVE(major,minor)
64  * TAG-DIRECTIVE(handle,prefix)
65  *
66  * Example:
67  *
68  * %YAML 1.1
69  * %TAG ! !foo
70  * %TAG !yaml! tag:yaml.org,2002:
71  * ---
72  *
73  * The corresponding sequence of tokens:
74  *
75  * STREAM-START(utf-8)
76  * VERSION-DIRECTIVE(1,1)
77  * TAG-DIRECTIVE("!","!foo")
78  * TAG-DIRECTIVE("!yaml","tag:yaml.org,2002:")
79  * DOCUMENT-START
80  * STREAM-END
81  *
82  * Note that the VERSION-DIRECTIVE and TAG-DIRECTIVE tokens occupy a whole
83  * line.
84  *
85  * The document start and end indicators are represented by:
86  *
87  * DOCUMENT-START
88  * DOCUMENT-END
89  *
90  * Note that if a YAML stream contains an implicit document (without '---'
91  * and '...' indicators), no DOCUMENT-START and DOCUMENT-END tokens will be
92  * produced.
93  *
94  * In the following examples, we present whole documents together with the
95  * produced tokens.
96  *
97  * 1. An implicit document:
98  *
99  * 'a scalar'
100  *
101  * Tokens:
102  *
103  * STREAM-START(utf-8)
104  * SCALAR("a scalar",single-quoted)
105  * STREAM-END
106  *
107  * 2. An explicit document:
108  *
109  * ---
110  * 'a scalar'
111  * ...
112  *
113  * Tokens:
114  *
115  * STREAM-START(utf-8)
116  * DOCUMENT-START
117  * SCALAR("a scalar",single-quoted)
118  * DOCUMENT-END
119  * STREAM-END
120  *
121  * 3. Several documents in a stream:
122  *
123  * 'a scalar'
124  * ---
125  * 'another scalar'
126  * ---
127  * 'yet another scalar'
128  *
129  * Tokens:
130  *
131  * STREAM-START(utf-8)
132  * SCALAR("a scalar",single-quoted)
133  * DOCUMENT-START
134  * SCALAR("another scalar",single-quoted)
135  * DOCUMENT-START
136  * SCALAR("yet another scalar",single-quoted)
137  * STREAM-END
138  *
139  * We have already introduced the SCALAR token above. The following tokens are
140  * used to describe aliases, anchors, tag, and scalars:
141  *
142  * ALIAS(anchor)
143  * ANCHOR(anchor)
144  * TAG(handle,suffix)
145  * SCALAR(value,style)
146  *
147  * The following series of examples illustrate the usage of these tokens:
148  *
149  * 1. A recursive sequence:
150  *
151  * &A [ *A ]
152  *
153  * Tokens:
154  *
155  * STREAM-START(utf-8)
156  * ANCHOR("A")
157  * FLOW-SEQUENCE-START
158  * ALIAS("A")
159  * FLOW-SEQUENCE-END
160  * STREAM-END
161  *
162  * 2. A tagged scalar:
163  *
164  * !!float "3.14" # A good approximation.
165  *
166  * Tokens:
167  *
168  * STREAM-START(utf-8)
169  * TAG("!!","float")
170  * SCALAR("3.14",double-quoted)
171  * STREAM-END
172  *
173  * 3. Various scalar styles:
174  *
175  * --- # Implicit empty plain scalars do not produce tokens.
176  * --- a plain scalar
177  * --- 'a single-quoted scalar'
178  * --- "a double-quoted scalar"
179  * --- |-
180  * a literal scalar
181  * --- >-
182  * a folded
183  * scalar
184  *
185  * Tokens:
186  *
187  * STREAM-START(utf-8)
188  * DOCUMENT-START
189  * DOCUMENT-START
190  * SCALAR("a plain scalar",plain)
191  * DOCUMENT-START
192  * SCALAR("a single-quoted scalar",single-quoted)
193  * DOCUMENT-START
194  * SCALAR("a double-quoted scalar",double-quoted)
195  * DOCUMENT-START
196  * SCALAR("a literal scalar",literal)
197  * DOCUMENT-START
198  * SCALAR("a folded scalar",folded)
199  * STREAM-END
200  *
201  * Now it's time to review collection-related tokens. We will start with
202  * flow collections:
203  *
204  * FLOW-SEQUENCE-START
205  * FLOW-SEQUENCE-END
206  * FLOW-MAPPING-START
207  * FLOW-MAPPING-END
208  * FLOW-ENTRY
209  * KEY
210  * VALUE
211  *
212  * The tokens FLOW-SEQUENCE-START, FLOW-SEQUENCE-END, FLOW-MAPPING-START, and
213  * FLOW-MAPPING-END represent the indicators '[', ']', '{', and '}'
214  * correspondingly. FLOW-ENTRY represent the ',' indicator. Finally the
215  * indicators '?' and ':', which are used for denoting mapping keys and values,
216  * are represented by the KEY and VALUE tokens.
217  *
218  * The following examples show flow collections:
219  *
220  * 1. A flow sequence:
221  *
222  * [item 1, item 2, item 3]
223  *
224  * Tokens:
225  *
226  * STREAM-START(utf-8)
227  * FLOW-SEQUENCE-START
228  * SCALAR("item 1",plain)
229  * FLOW-ENTRY
230  * SCALAR("item 2",plain)
231  * FLOW-ENTRY
232  * SCALAR("item 3",plain)
233  * FLOW-SEQUENCE-END
234  * STREAM-END
235  *
236  * 2. A flow mapping:
237  *
238  * {
239  * a simple key: a value, # Note that the KEY token is produced.
240  * ? a complex key: another value,
241  * }
242  *
243  * Tokens:
244  *
245  * STREAM-START(utf-8)
246  * FLOW-MAPPING-START
247  * KEY
248  * SCALAR("a simple key",plain)
249  * VALUE
250  * SCALAR("a value",plain)
251  * FLOW-ENTRY
252  * KEY
253  * SCALAR("a complex key",plain)
254  * VALUE
255  * SCALAR("another value",plain)
256  * FLOW-ENTRY
257  * FLOW-MAPPING-END
258  * STREAM-END
259  *
260  * A simple key is a key which is not denoted by the '?' indicator. Note that
261  * the Scanner still produce the KEY token whenever it encounters a simple key.
262  *
263  * For scanning block collections, the following tokens are used (note that we
264  * repeat KEY and VALUE here):
265  *
266  * BLOCK-SEQUENCE-START
267  * BLOCK-MAPPING-START
268  * BLOCK-END
269  * BLOCK-ENTRY
270  * KEY
271  * VALUE
272  *
273  * The tokens BLOCK-SEQUENCE-START and BLOCK-MAPPING-START denote indentation
274  * increase that precedes a block collection (cf. the INDENT token in Python).
275  * The token BLOCK-END denote indentation decrease that ends a block collection
276  * (cf. the DEDENT token in Python). However YAML has some syntax pecularities
277  * that makes detections of these tokens more complex.
278  *
279  * The tokens BLOCK-ENTRY, KEY, and VALUE are used to represent the indicators
280  * '-', '?', and ':' correspondingly.
281  *
282  * The following examples show how the tokens BLOCK-SEQUENCE-START,
283  * BLOCK-MAPPING-START, and BLOCK-END are emitted by the Scanner:
284  *
285  * 1. Block sequences:
286  *
287  * - item 1
288  * - item 2
289  * -
290  * - item 3.1
291  * - item 3.2
292  * -
293  * key 1: value 1
294  * key 2: value 2
295  *
296  * Tokens:
297  *
298  * STREAM-START(utf-8)
299  * BLOCK-SEQUENCE-START
300  * BLOCK-ENTRY
301  * SCALAR("item 1",plain)
302  * BLOCK-ENTRY
303  * SCALAR("item 2",plain)
304  * BLOCK-ENTRY
305  * BLOCK-SEQUENCE-START
306  * BLOCK-ENTRY
307  * SCALAR("item 3.1",plain)
308  * BLOCK-ENTRY
309  * SCALAR("item 3.2",plain)
310  * BLOCK-END
311  * BLOCK-ENTRY
312  * BLOCK-MAPPING-START
313  * KEY
314  * SCALAR("key 1",plain)
315  * VALUE
316  * SCALAR("value 1",plain)
317  * KEY
318  * SCALAR("key 2",plain)
319  * VALUE
320  * SCALAR("value 2",plain)
321  * BLOCK-END
322  * BLOCK-END
323  * STREAM-END
324  *
325  * 2. Block mappings:
326  *
327  * a simple key: a value # The KEY token is produced here.
328  * ? a complex key
329  * : another value
330  * a mapping:
331  * key 1: value 1
332  * key 2: value 2
333  * a sequence:
334  * - item 1
335  * - item 2
336  *
337  * Tokens:
338  *
339  * STREAM-START(utf-8)
340  * BLOCK-MAPPING-START
341  * KEY
342  * SCALAR("a simple key",plain)
343  * VALUE
344  * SCALAR("a value",plain)
345  * KEY
346  * SCALAR("a complex key",plain)
347  * VALUE
348  * SCALAR("another value",plain)
349  * KEY
350  * SCALAR("a mapping",plain)
351  * BLOCK-MAPPING-START
352  * KEY
353  * SCALAR("key 1",plain)
354  * VALUE
355  * SCALAR("value 1",plain)
356  * KEY
357  * SCALAR("key 2",plain)
358  * VALUE
359  * SCALAR("value 2",plain)
360  * BLOCK-END
361  * KEY
362  * SCALAR("a sequence",plain)
363  * VALUE
364  * BLOCK-SEQUENCE-START
365  * BLOCK-ENTRY
366  * SCALAR("item 1",plain)
367  * BLOCK-ENTRY
368  * SCALAR("item 2",plain)
369  * BLOCK-END
370  * BLOCK-END
371  * STREAM-END
372  *
373  * YAML does not always require to start a new block collection from a new
374  * line. If the current line contains only '-', '?', and ':' indicators, a new
375  * block collection may start at the current line. The following examples
376  * illustrate this case:
377  *
378  * 1. Collections in a sequence:
379  *
380  * - - item 1
381  * - item 2
382  * - key 1: value 1
383  * key 2: value 2
384  * - ? complex key
385  * : complex value
386  *
387  * Tokens:
388  *
389  * STREAM-START(utf-8)
390  * BLOCK-SEQUENCE-START
391  * BLOCK-ENTRY
392  * BLOCK-SEQUENCE-START
393  * BLOCK-ENTRY
394  * SCALAR("item 1",plain)
395  * BLOCK-ENTRY
396  * SCALAR("item 2",plain)
397  * BLOCK-END
398  * BLOCK-ENTRY
399  * BLOCK-MAPPING-START
400  * KEY
401  * SCALAR("key 1",plain)
402  * VALUE
403  * SCALAR("value 1",plain)
404  * KEY
405  * SCALAR("key 2",plain)
406  * VALUE
407  * SCALAR("value 2",plain)
408  * BLOCK-END
409  * BLOCK-ENTRY
410  * BLOCK-MAPPING-START
411  * KEY
412  * SCALAR("complex key")
413  * VALUE
414  * SCALAR("complex value")
415  * BLOCK-END
416  * BLOCK-END
417  * STREAM-END
418  *
419  * 2. Collections in a mapping:
420  *
421  * ? a sequence
422  * : - item 1
423  * - item 2
424  * ? a mapping
425  * : key 1: value 1
426  * key 2: value 2
427  *
428  * Tokens:
429  *
430  * STREAM-START(utf-8)
431  * BLOCK-MAPPING-START
432  * KEY
433  * SCALAR("a sequence",plain)
434  * VALUE
435  * BLOCK-SEQUENCE-START
436  * BLOCK-ENTRY
437  * SCALAR("item 1",plain)
438  * BLOCK-ENTRY
439  * SCALAR("item 2",plain)
440  * BLOCK-END
441  * KEY
442  * SCALAR("a mapping",plain)
443  * VALUE
444  * BLOCK-MAPPING-START
445  * KEY
446  * SCALAR("key 1",plain)
447  * VALUE
448  * SCALAR("value 1",plain)
449  * KEY
450  * SCALAR("key 2",plain)
451  * VALUE
452  * SCALAR("value 2",plain)
453  * BLOCK-END
454  * BLOCK-END
455  * STREAM-END
456  *
457  * YAML also permits non-indented sequences if they are included into a block
458  * mapping. In this case, the token BLOCK-SEQUENCE-START is not produced:
459  *
460  * key:
461  * - item 1 # BLOCK-SEQUENCE-START is NOT produced here.
462  * - item 2
463  *
464  * Tokens:
465  *
466  * STREAM-START(utf-8)
467  * BLOCK-MAPPING-START
468  * KEY
469  * SCALAR("key",plain)
470  * VALUE
471  * BLOCK-ENTRY
472  * SCALAR("item 1",plain)
473  * BLOCK-ENTRY
474  * SCALAR("item 2",plain)
475  * BLOCK-END
476  */
477 
478 #include "yaml_private.h"
479 
480 /*
481  * Ensure that the buffer contains the required number of characters.
482  * Return 1 on success, 0 on failure (reader error or memory error).
483  */
484 
485 #define CACHE(parser,length) \
486  (parser->unread >= (length) \
487  ? 1 \
488  : yaml_parser_update_buffer(parser, (length)))
489 
490 /*
491  * Advance the buffer pointer.
492  */
493 
494 #define SKIP(parser) \
495  (parser->mark.index ++, \
496  parser->mark.column ++, \
497  parser->unread --, \
498  parser->buffer.pointer += WIDTH(parser->buffer))
499 
500 #define SKIP_LINE(parser) \
501  (IS_CRLF(parser->buffer) ? \
502  (parser->mark.index += 2, \
503  parser->mark.column = 0, \
504  parser->mark.line ++, \
505  parser->unread -= 2, \
506  parser->buffer.pointer += 2) : \
507  IS_BREAK(parser->buffer) ? \
508  (parser->mark.index ++, \
509  parser->mark.column = 0, \
510  parser->mark.line ++, \
511  parser->unread --, \
512  parser->buffer.pointer += WIDTH(parser->buffer)) : 0)
513 
514 /*
515  * Copy a character to a string buffer and advance pointers.
516  */
517 
518 #define READ(parser,string) \
519  (STRING_EXTEND(parser,string) ? \
520  (COPY(string,parser->buffer), \
521  parser->mark.index ++, \
522  parser->mark.column ++, \
523  parser->unread --, \
524  1) : 0)
525 
526 /*
527  * Copy a line break character to a string buffer and advance pointers.
528  */
529 
530 #define READ_LINE(parser,string) \
531  (STRING_EXTEND(parser,string) ? \
532  (((CHECK_AT(parser->buffer,'\r',0) \
533  && CHECK_AT(parser->buffer,'\n',1)) ? /* CR LF -> LF */ \
534  (*((string).pointer++) = (yaml_char_t) '\n', \
535  parser->buffer.pointer += 2, \
536  parser->mark.index += 2, \
537  parser->mark.column = 0, \
538  parser->mark.line ++, \
539  parser->unread -= 2) : \
540  (CHECK_AT(parser->buffer,'\r',0) \
541  || CHECK_AT(parser->buffer,'\n',0)) ? /* CR|LF -> LF */ \
542  (*((string).pointer++) = (yaml_char_t) '\n', \
543  parser->buffer.pointer ++, \
544  parser->mark.index ++, \
545  parser->mark.column = 0, \
546  parser->mark.line ++, \
547  parser->unread --) : \
548  (CHECK_AT(parser->buffer,'\xC2',0) \
549  && CHECK_AT(parser->buffer,'\x85',1)) ? /* NEL -> LF */ \
550  (*((string).pointer++) = (yaml_char_t) '\n', \
551  parser->buffer.pointer += 2, \
552  parser->mark.index ++, \
553  parser->mark.column = 0, \
554  parser->mark.line ++, \
555  parser->unread --) : \
556  (CHECK_AT(parser->buffer,'\xE2',0) && \
557  CHECK_AT(parser->buffer,'\x80',1) && \
558  (CHECK_AT(parser->buffer,'\xA8',2) || \
559  CHECK_AT(parser->buffer,'\xA9',2))) ? /* LS|PS -> LS|PS */ \
560  (*((string).pointer++) = *(parser->buffer.pointer++), \
561  *((string).pointer++) = *(parser->buffer.pointer++), \
562  *((string).pointer++) = *(parser->buffer.pointer++), \
563  parser->mark.index ++, \
564  parser->mark.column = 0, \
565  parser->mark.line ++, \
566  parser->unread --) : 0), \
567  1) : 0)
568 
569 /*
570  * Public API declarations.
571  */
572 
573 YAML_DECLARE(int)
575 
576 /*
577  * Error handling.
578  */
579 
580 static int
581 yaml_parser_set_scanner_error(yaml_parser_t *parser, const char *context,
582  yaml_mark_t context_mark, const char *problem);
583 
584 /*
585  * High-level token API.
586  */
587 
588 YAML_DECLARE(int)
590 
591 static int
593 
594 /*
595  * Potential simple keys.
596  */
597 
598 static int
600 
601 static int
603 
604 static int
606 
607 static int
609 
610 static int
612 
613 /*
614  * Indentation treatment.
615  */
616 
617 static int
618 yaml_parser_roll_indent(yaml_parser_t *parser, ptrdiff_t column,
619  ptrdiff_t number, yaml_token_type_t type, yaml_mark_t mark);
620 
621 static int
622 yaml_parser_unroll_indent(yaml_parser_t *parser, ptrdiff_t column);
623 
624 /*
625  * Token fetchers.
626  */
627 
628 static int
630 
631 static int
633 
634 static int
636 
637 static int
639  yaml_token_type_t type);
640 
641 static int
643  yaml_token_type_t type);
644 
645 static int
647  yaml_token_type_t type);
648 
649 static int
651 
652 static int
654 
655 static int
657 
658 static int
660 
661 static int
663 
664 static int
666 
667 static int
668 yaml_parser_fetch_block_scalar(yaml_parser_t *parser, int literal);
669 
670 static int
671 yaml_parser_fetch_flow_scalar(yaml_parser_t *parser, int single);
672 
673 static int
675 
676 /*
677  * Token scanners.
678  */
679 
680 static int
682 
683 static int
685 
686 static int
688  yaml_mark_t start_mark, yaml_char_t **name);
689 
690 static int
692  yaml_mark_t start_mark, int *major, int *minor);
693 
694 static int
696  yaml_mark_t start_mark, int *number);
697 
698 static int
700  yaml_mark_t mark, yaml_char_t **handle, yaml_char_t **prefix);
701 
702 static int
704  yaml_token_type_t type);
705 
706 static int
708 
709 static int
710 yaml_parser_scan_tag_handle(yaml_parser_t *parser, int directive,
711  yaml_mark_t start_mark, yaml_char_t **handle);
712 
713 static int
714 yaml_parser_scan_tag_uri(yaml_parser_t *parser, int directive,
715  yaml_char_t *head, yaml_mark_t start_mark, yaml_char_t **uri);
716 
717 static int
718 yaml_parser_scan_uri_escapes(yaml_parser_t *parser, int directive,
719  yaml_mark_t start_mark, yaml_string_t *string);
720 
721 static int
723  int literal);
724 
725 static int
727  int *indent, yaml_string_t *breaks,
728  yaml_mark_t start_mark, yaml_mark_t *end_mark);
729 
730 static int
732  int single);
733 
734 static int
736 
737 /*
738  * Get the next token.
739  */
740 
741 YAML_DECLARE(int)
743 {
744  assert(parser); /* Non-NULL parser object is expected. */
745  assert(token); /* Non-NULL token object is expected. */
746 
747  /* Erase the token object. */
748 
749  memset(token, 0, sizeof(yaml_token_t));
750 
751  /* No tokens after STREAM-END or error. */
752 
753  if (parser->stream_end_produced || parser->error) {
754  return 1;
755  }
756 
757  /* Ensure that the tokens queue contains enough tokens. */
758 
759  if (!parser->token_available) {
760  if (!yaml_parser_fetch_more_tokens(parser))
761  return 0;
762  }
763 
764  /* Fetch the next token from the queue. */
765 
766  *token = DEQUEUE(parser, parser->tokens);
767  parser->token_available = 0;
768  parser->tokens_parsed ++;
769 
770  if (token->type == YAML_STREAM_END_TOKEN) {
771  parser->stream_end_produced = 1;
772  }
773 
774  return 1;
775 }
776 
777 /*
778  * Set the scanner error and return 0.
779  */
780 
781 static int
782 yaml_parser_set_scanner_error(yaml_parser_t *parser, const char *context,
783  yaml_mark_t context_mark, const char *problem)
784 {
785  parser->error = YAML_SCANNER_ERROR;
786  parser->context = context;
787  parser->context_mark = context_mark;
788  parser->problem = problem;
789  parser->problem_mark = parser->mark;
790 
791  return 0;
792 }
793 
794 /*
795  * Ensure that the tokens queue contains at least one token which can be
796  * returned to the Parser.
797  */
798 
799 YAML_DECLARE(int)
801 {
802  int need_more_tokens;
803 
804  /* While we need more tokens to fetch, do it. */
805 
806  while (1)
807  {
808  /*
809  * Check if we really need to fetch more tokens.
810  */
811 
812  need_more_tokens = 0;
813 
814  if (parser->tokens.head == parser->tokens.tail)
815  {
816  /* Queue is empty. */
817 
818  need_more_tokens = 1;
819  }
820  else
821  {
822  yaml_simple_key_t *simple_key;
823 
824  /* Check if any potential simple key may occupy the head position. */
825 
826  if (!yaml_parser_stale_simple_keys(parser))
827  return 0;
828 
829  for (simple_key = parser->simple_keys.start;
830  simple_key != parser->simple_keys.top; simple_key++) {
831  if (simple_key->possible
832  && simple_key->token_number == parser->tokens_parsed) {
833  need_more_tokens = 1;
834  break;
835  }
836  }
837  }
838 
839  /* We are finished. */
840 
841  if (!need_more_tokens)
842  break;
843 
844  /* Fetch the next token. */
845 
846  if (!yaml_parser_fetch_next_token(parser))
847  return 0;
848  }
849 
850  parser->token_available = 1;
851 
852  return 1;
853 }
854 
855 /*
856  * The dispatcher for token fetchers.
857  */
858 
859 static int
861 {
862  /* Ensure that the buffer is initialized. */
863 
864  if (!CACHE(parser, 1))
865  return 0;
866 
867  /* Check if we just started scanning. Fetch STREAM-START then. */
868 
869  if (!parser->stream_start_produced)
870  return yaml_parser_fetch_stream_start(parser);
871 
872  /* Eat whitespaces and comments until we reach the next token. */
873 
874  if (!yaml_parser_scan_to_next_token(parser))
875  return 0;
876 
877  /* Remove obsolete potential simple keys. */
878 
879  if (!yaml_parser_stale_simple_keys(parser))
880  return 0;
881 
882  /* Check the indentation level against the current column. */
883 
884  if (!yaml_parser_unroll_indent(parser, parser->mark.column))
885  return 0;
886 
887  /*
888  * Ensure that the buffer contains at least 4 characters. 4 is the length
889  * of the longest indicators ('--- ' and '... ').
890  */
891 
892  if (!CACHE(parser, 4))
893  return 0;
894 
895  /* Is it the end of the stream? */
896 
897  if (IS_Z(parser->buffer))
898  return yaml_parser_fetch_stream_end(parser);
899 
900  /* Is it a directive? */
901 
902  if (parser->mark.column == 0 && CHECK(parser->buffer, '%'))
903  return yaml_parser_fetch_directive(parser);
904 
905  /* Is it the document start indicator? */
906 
907  if (parser->mark.column == 0
908  && CHECK_AT(parser->buffer, '-', 0)
909  && CHECK_AT(parser->buffer, '-', 1)
910  && CHECK_AT(parser->buffer, '-', 2)
911  && IS_BLANKZ_AT(parser->buffer, 3))
914 
915  /* Is it the document end indicator? */
916 
917  if (parser->mark.column == 0
918  && CHECK_AT(parser->buffer, '.', 0)
919  && CHECK_AT(parser->buffer, '.', 1)
920  && CHECK_AT(parser->buffer, '.', 2)
921  && IS_BLANKZ_AT(parser->buffer, 3))
924 
925  /* Is it the flow sequence start indicator? */
926 
927  if (CHECK(parser->buffer, '['))
930 
931  /* Is it the flow mapping start indicator? */
932 
933  if (CHECK(parser->buffer, '{'))
936 
937  /* Is it the flow sequence end indicator? */
938 
939  if (CHECK(parser->buffer, ']'))
942 
943  /* Is it the flow mapping end indicator? */
944 
945  if (CHECK(parser->buffer, '}'))
948 
949  /* Is it the flow entry indicator? */
950 
951  if (CHECK(parser->buffer, ','))
952  return yaml_parser_fetch_flow_entry(parser);
953 
954  /* Is it the block entry indicator? */
955 
956  if (CHECK(parser->buffer, '-') && IS_BLANKZ_AT(parser->buffer, 1))
957  return yaml_parser_fetch_block_entry(parser);
958 
959  /* Is it the key indicator? */
960 
961  if (CHECK(parser->buffer, '?')
962  && (parser->flow_level || IS_BLANKZ_AT(parser->buffer, 1)))
963  return yaml_parser_fetch_key(parser);
964 
965  /* Is it the value indicator? */
966 
967  if (CHECK(parser->buffer, ':')
968  && (parser->flow_level || IS_BLANKZ_AT(parser->buffer, 1)))
969  return yaml_parser_fetch_value(parser);
970 
971  /* Is it an alias? */
972 
973  if (CHECK(parser->buffer, '*'))
975 
976  /* Is it an anchor? */
977 
978  if (CHECK(parser->buffer, '&'))
980 
981  /* Is it a tag? */
982 
983  if (CHECK(parser->buffer, '!'))
984  return yaml_parser_fetch_tag(parser);
985 
986  /* Is it a literal scalar? */
987 
988  if (CHECK(parser->buffer, '|') && !parser->flow_level)
989  return yaml_parser_fetch_block_scalar(parser, 1);
990 
991  /* Is it a folded scalar? */
992 
993  if (CHECK(parser->buffer, '>') && !parser->flow_level)
994  return yaml_parser_fetch_block_scalar(parser, 0);
995 
996  /* Is it a single-quoted scalar? */
997 
998  if (CHECK(parser->buffer, '\''))
999  return yaml_parser_fetch_flow_scalar(parser, 1);
1000 
1001  /* Is it a double-quoted scalar? */
1002 
1003  if (CHECK(parser->buffer, '"'))
1004  return yaml_parser_fetch_flow_scalar(parser, 0);
1005 
1006  /*
1007  * Is it a plain scalar?
1008  *
1009  * A plain scalar may start with any non-blank characters except
1010  *
1011  * '-', '?', ':', ',', '[', ']', '{', '}',
1012  * '#', '&', '*', '!', '|', '>', '\'', '\"',
1013  * '%', '@', '`'.
1014  *
1015  * In the block context (and, for the '-' indicator, in the flow context
1016  * too), it may also start with the characters
1017  *
1018  * '-', '?', ':'
1019  *
1020  * if it is followed by a non-space character.
1021  *
1022  * The last rule is more restrictive than the specification requires.
1023  */
1024 
1025  if (!(IS_BLANKZ(parser->buffer) || CHECK(parser->buffer, '-')
1026  || CHECK(parser->buffer, '?') || CHECK(parser->buffer, ':')
1027  || CHECK(parser->buffer, ',') || CHECK(parser->buffer, '[')
1028  || CHECK(parser->buffer, ']') || CHECK(parser->buffer, '{')
1029  || CHECK(parser->buffer, '}') || CHECK(parser->buffer, '#')
1030  || CHECK(parser->buffer, '&') || CHECK(parser->buffer, '*')
1031  || CHECK(parser->buffer, '!') || CHECK(parser->buffer, '|')
1032  || CHECK(parser->buffer, '>') || CHECK(parser->buffer, '\'')
1033  || CHECK(parser->buffer, '"') || CHECK(parser->buffer, '%')
1034  || CHECK(parser->buffer, '@') || CHECK(parser->buffer, '`')) ||
1035  (CHECK(parser->buffer, '-') && !IS_BLANK_AT(parser->buffer, 1)) ||
1036  (!parser->flow_level &&
1037  (CHECK(parser->buffer, '?') || CHECK(parser->buffer, ':'))
1038  && !IS_BLANKZ_AT(parser->buffer, 1)))
1039  return yaml_parser_fetch_plain_scalar(parser);
1040 
1041  /*
1042  * If we don't determine the token type so far, it is an error.
1043  */
1044 
1045  return yaml_parser_set_scanner_error(parser,
1046  "while scanning for the next token", parser->mark,
1047  "found character that cannot start any token");
1048 }
1049 
1050 /*
1051  * Check the list of potential simple keys and remove the positions that
1052  * cannot contain simple keys anymore.
1053  */
1054 
1055 static int
1057 {
1058  yaml_simple_key_t *simple_key;
1059 
1060  /* Check for a potential simple key for each flow level. */
1061 
1062  for (simple_key = parser->simple_keys.start;
1063  simple_key != parser->simple_keys.top; simple_key ++)
1064  {
1065  /*
1066  * The specification requires that a simple key
1067  *
1068  * - is limited to a single line,
1069  * - is shorter than 1024 characters.
1070  */
1071 
1072  if (simple_key->possible
1073  && (simple_key->mark.line < parser->mark.line
1074  || simple_key->mark.index+1024 < parser->mark.index)) {
1075 
1076  /* Check if the potential simple key to be removed is required. */
1077 
1078  if (simple_key->required) {
1079  return yaml_parser_set_scanner_error(parser,
1080  "while scanning a simple key", simple_key->mark,
1081  "could not find expected ':'");
1082  }
1083 
1084  simple_key->possible = 0;
1085  }
1086  }
1087 
1088  return 1;
1089 }
1090 
1091 /*
1092  * Check if a simple key may start at the current position and add it if
1093  * needed.
1094  */
1095 
1096 static int
1098 {
1099  /*
1100  * A simple key is required at the current position if the scanner is in
1101  * the block context and the current column coincides with the indentation
1102  * level.
1103  */
1104 
1105  int required = (!parser->flow_level
1106  && parser->indent == (ptrdiff_t)parser->mark.column);
1107 
1108  /*
1109  * If the current position may start a simple key, save it.
1110  */
1111 
1112  if (parser->simple_key_allowed)
1113  {
1114  yaml_simple_key_t simple_key;
1115  simple_key.possible = 1;
1116  simple_key.required = required;
1117  simple_key.token_number =
1118  parser->tokens_parsed + (parser->tokens.tail - parser->tokens.head);
1119  simple_key.mark = parser->mark;
1120 
1121  if (!yaml_parser_remove_simple_key(parser)) return 0;
1122 
1123  *(parser->simple_keys.top-1) = simple_key;
1124  }
1125 
1126  return 1;
1127 }
1128 
1129 /*
1130  * Remove a potential simple key at the current flow level.
1131  */
1132 
1133 static int
1135 {
1136  yaml_simple_key_t *simple_key = parser->simple_keys.top-1;
1137 
1138  if (simple_key->possible)
1139  {
1140  /* If the key is required, it is an error. */
1141 
1142  if (simple_key->required) {
1143  return yaml_parser_set_scanner_error(parser,
1144  "while scanning a simple key", simple_key->mark,
1145  "could not find expected ':'");
1146  }
1147  }
1148 
1149  /* Remove the key from the stack. */
1150 
1151  simple_key->possible = 0;
1152 
1153  return 1;
1154 }
1155 
1156 /*
1157  * Increase the flow level and resize the simple key list if needed.
1158  */
1159 
1160 static int
1162 {
1163  yaml_simple_key_t empty_simple_key = { 0, 0, 0, { 0, 0, 0 } };
1164 
1165  /* Reset the simple key on the next level. */
1166 
1167  if (!PUSH(parser, parser->simple_keys, empty_simple_key))
1168  return 0;
1169 
1170  /* Increase the flow level. */
1171 
1172  if (parser->flow_level == INT_MAX) {
1173  parser->error = YAML_MEMORY_ERROR;
1174  return 0;
1175  }
1176 
1177  parser->flow_level++;
1178 
1179  return 1;
1180 }
1181 
1182 /*
1183  * Decrease the flow level.
1184  */
1185 
1186 static int
1188 {
1189  yaml_simple_key_t dummy_key; /* Used to eliminate a compiler warning. */
1190 
1191  if (parser->flow_level) {
1192  parser->flow_level --;
1193  dummy_key = POP(parser, parser->simple_keys);
1194  }
1195 
1196  return 1;
1197 }
1198 
1199 /*
1200  * Push the current indentation level to the stack and set the new level
1201  * the current column is greater than the indentation level. In this case,
1202  * append or insert the specified token into the token queue.
1203  *
1204  */
1205 
1206 static int
1207 yaml_parser_roll_indent(yaml_parser_t *parser, ptrdiff_t column,
1208  ptrdiff_t number, yaml_token_type_t type, yaml_mark_t mark)
1209 {
1211 
1212  /* In the flow context, do nothing. */
1213 
1214  if (parser->flow_level)
1215  return 1;
1216 
1217  if (parser->indent < column)
1218  {
1219  /*
1220  * Push the current indentation level to the stack and set the new
1221  * indentation level.
1222  */
1223 
1224  if (!PUSH(parser, parser->indents, parser->indent))
1225  return 0;
1226 
1227  if (column > INT_MAX) {
1228  parser->error = YAML_MEMORY_ERROR;
1229  return 0;
1230  }
1231 
1232  parser->indent = column;
1233 
1234  /* Create a token and insert it into the queue. */
1235 
1236  TOKEN_INIT(token, type, mark, mark);
1237 
1238  if (number == -1) {
1239  if (!ENQUEUE(parser, parser->tokens, token))
1240  return 0;
1241  }
1242  else {
1243  if (!QUEUE_INSERT(parser,
1244  parser->tokens, number - parser->tokens_parsed, token))
1245  return 0;
1246  }
1247  }
1248 
1249  return 1;
1250 }
1251 
1252 /*
1253  * Pop indentation levels from the indents stack until the current level
1254  * becomes less or equal to the column. For each indentation level, append
1255  * the BLOCK-END token.
1256  */
1257 
1258 
1259 static int
1260 yaml_parser_unroll_indent(yaml_parser_t *parser, ptrdiff_t column)
1261 {
1263 
1264  /* In the flow context, do nothing. */
1265 
1266  if (parser->flow_level)
1267  return 1;
1268 
1269  /* Loop through the indentation levels in the stack. */
1270 
1271  while (parser->indent > column)
1272  {
1273  /* Create a token and append it to the queue. */
1274 
1275  TOKEN_INIT(token, YAML_BLOCK_END_TOKEN, parser->mark, parser->mark);
1276 
1277  if (!ENQUEUE(parser, parser->tokens, token))
1278  return 0;
1279 
1280  /* Pop the indentation level. */
1281 
1282  parser->indent = POP(parser, parser->indents);
1283  }
1284 
1285  return 1;
1286 }
1287 
1288 /*
1289  * Initialize the scanner and produce the STREAM-START token.
1290  */
1291 
1292 static int
1294 {
1295  yaml_simple_key_t simple_key = { 0, 0, 0, { 0, 0, 0 } };
1297 
1298  /* Set the initial indentation. */
1299 
1300  parser->indent = -1;
1301 
1302  /* Initialize the simple key stack. */
1303 
1304  if (!PUSH(parser, parser->simple_keys, simple_key))
1305  return 0;
1306 
1307  /* A simple key is allowed at the beginning of the stream. */
1308 
1309  parser->simple_key_allowed = 1;
1310 
1311  /* We have started. */
1312 
1313  parser->stream_start_produced = 1;
1314 
1315  /* Create the STREAM-START token and append it to the queue. */
1316 
1317  STREAM_START_TOKEN_INIT(token, parser->encoding,
1318  parser->mark, parser->mark);
1319 
1320  if (!ENQUEUE(parser, parser->tokens, token))
1321  return 0;
1322 
1323  return 1;
1324 }
1325 
1326 /*
1327  * Produce the STREAM-END token and shut down the scanner.
1328  */
1329 
1330 static int
1332 {
1334 
1335  /* Force new line. */
1336 
1337  if (parser->mark.column != 0) {
1338  parser->mark.column = 0;
1339  parser->mark.line ++;
1340  }
1341 
1342  /* Reset the indentation level. */
1343 
1344  if (!yaml_parser_unroll_indent(parser, -1))
1345  return 0;
1346 
1347  /* Reset simple keys. */
1348 
1349  if (!yaml_parser_remove_simple_key(parser))
1350  return 0;
1351 
1352  parser->simple_key_allowed = 0;
1353 
1354  /* Create the STREAM-END token and append it to the queue. */
1355 
1356  STREAM_END_TOKEN_INIT(token, parser->mark, parser->mark);
1357 
1358  if (!ENQUEUE(parser, parser->tokens, token))
1359  return 0;
1360 
1361  return 1;
1362 }
1363 
1364 /*
1365  * Produce a VERSION-DIRECTIVE or TAG-DIRECTIVE token.
1366  */
1367 
1368 static int
1370 {
1372 
1373  /* Reset the indentation level. */
1374 
1375  if (!yaml_parser_unroll_indent(parser, -1))
1376  return 0;
1377 
1378  /* Reset simple keys. */
1379 
1380  if (!yaml_parser_remove_simple_key(parser))
1381  return 0;
1382 
1383  parser->simple_key_allowed = 0;
1384 
1385  /* Create the YAML-DIRECTIVE or TAG-DIRECTIVE token. */
1386 
1387  if (!yaml_parser_scan_directive(parser, &token))
1388  return 0;
1389 
1390  /* Append the token to the queue. */
1391 
1392  if (!ENQUEUE(parser, parser->tokens, token)) {
1393  yaml_token_delete(&token);
1394  return 0;
1395  }
1396 
1397  return 1;
1398 }
1399 
1400 /*
1401  * Produce the DOCUMENT-START or DOCUMENT-END token.
1402  */
1403 
1404 static int
1406  yaml_token_type_t type)
1407 {
1408  yaml_mark_t start_mark, end_mark;
1410 
1411  /* Reset the indentation level. */
1412 
1413  if (!yaml_parser_unroll_indent(parser, -1))
1414  return 0;
1415 
1416  /* Reset simple keys. */
1417 
1418  if (!yaml_parser_remove_simple_key(parser))
1419  return 0;
1420 
1421  parser->simple_key_allowed = 0;
1422 
1423  /* Consume the token. */
1424 
1425  start_mark = parser->mark;
1426 
1427  SKIP(parser);
1428  SKIP(parser);
1429  SKIP(parser);
1430 
1431  end_mark = parser->mark;
1432 
1433  /* Create the DOCUMENT-START or DOCUMENT-END token. */
1434 
1435  TOKEN_INIT(token, type, start_mark, end_mark);
1436 
1437  /* Append the token to the queue. */
1438 
1439  if (!ENQUEUE(parser, parser->tokens, token))
1440  return 0;
1441 
1442  return 1;
1443 }
1444 
1445 /*
1446  * Produce the FLOW-SEQUENCE-START or FLOW-MAPPING-START token.
1447  */
1448 
1449 static int
1451  yaml_token_type_t type)
1452 {
1453  yaml_mark_t start_mark, end_mark;
1455 
1456  /* The indicators '[' and '{' may start a simple key. */
1457 
1458  if (!yaml_parser_save_simple_key(parser))
1459  return 0;
1460 
1461  /* Increase the flow level. */
1462 
1463  if (!yaml_parser_increase_flow_level(parser))
1464  return 0;
1465 
1466  /* A simple key may follow the indicators '[' and '{'. */
1467 
1468  parser->simple_key_allowed = 1;
1469 
1470  /* Consume the token. */
1471 
1472  start_mark = parser->mark;
1473  SKIP(parser);
1474  end_mark = parser->mark;
1475 
1476  /* Create the FLOW-SEQUENCE-START of FLOW-MAPPING-START token. */
1477 
1478  TOKEN_INIT(token, type, start_mark, end_mark);
1479 
1480  /* Append the token to the queue. */
1481 
1482  if (!ENQUEUE(parser, parser->tokens, token))
1483  return 0;
1484 
1485  return 1;
1486 }
1487 
1488 /*
1489  * Produce the FLOW-SEQUENCE-END or FLOW-MAPPING-END token.
1490  */
1491 
1492 static int
1494  yaml_token_type_t type)
1495 {
1496  yaml_mark_t start_mark, end_mark;
1498 
1499  /* Reset any potential simple key on the current flow level. */
1500 
1501  if (!yaml_parser_remove_simple_key(parser))
1502  return 0;
1503 
1504  /* Decrease the flow level. */
1505 
1506  if (!yaml_parser_decrease_flow_level(parser))
1507  return 0;
1508 
1509  /* No simple keys after the indicators ']' and '}'. */
1510 
1511  parser->simple_key_allowed = 0;
1512 
1513  /* Consume the token. */
1514 
1515  start_mark = parser->mark;
1516  SKIP(parser);
1517  end_mark = parser->mark;
1518 
1519  /* Create the FLOW-SEQUENCE-END of FLOW-MAPPING-END token. */
1520 
1521  TOKEN_INIT(token, type, start_mark, end_mark);
1522 
1523  /* Append the token to the queue. */
1524 
1525  if (!ENQUEUE(parser, parser->tokens, token))
1526  return 0;
1527 
1528  return 1;
1529 }
1530 
1531 /*
1532  * Produce the FLOW-ENTRY token.
1533  */
1534 
1535 static int
1537 {
1538  yaml_mark_t start_mark, end_mark;
1540 
1541  /* Reset any potential simple keys on the current flow level. */
1542 
1543  if (!yaml_parser_remove_simple_key(parser))
1544  return 0;
1545 
1546  /* Simple keys are allowed after ','. */
1547 
1548  parser->simple_key_allowed = 1;
1549 
1550  /* Consume the token. */
1551 
1552  start_mark = parser->mark;
1553  SKIP(parser);
1554  end_mark = parser->mark;
1555 
1556  /* Create the FLOW-ENTRY token and append it to the queue. */
1557 
1558  TOKEN_INIT(token, YAML_FLOW_ENTRY_TOKEN, start_mark, end_mark);
1559 
1560  if (!ENQUEUE(parser, parser->tokens, token))
1561  return 0;
1562 
1563  return 1;
1564 }
1565 
1566 /*
1567  * Produce the BLOCK-ENTRY token.
1568  */
1569 
1570 static int
1572 {
1573  yaml_mark_t start_mark, end_mark;
1575 
1576  /* Check if the scanner is in the block context. */
1577 
1578  if (!parser->flow_level)
1579  {
1580  /* Check if we are allowed to start a new entry. */
1581 
1582  if (!parser->simple_key_allowed) {
1583  return yaml_parser_set_scanner_error(parser, NULL, parser->mark,
1584  "block sequence entries are not allowed in this context");
1585  }
1586 
1587  /* Add the BLOCK-SEQUENCE-START token if needed. */
1588 
1589  if (!yaml_parser_roll_indent(parser, parser->mark.column, -1,
1591  return 0;
1592  }
1593  else
1594  {
1595  /*
1596  * It is an error for the '-' indicator to occur in the flow context,
1597  * but we let the Parser detect and report about it because the Parser
1598  * is able to point to the context.
1599  */
1600  }
1601 
1602  /* Reset any potential simple keys on the current flow level. */
1603 
1604  if (!yaml_parser_remove_simple_key(parser))
1605  return 0;
1606 
1607  /* Simple keys are allowed after '-'. */
1608 
1609  parser->simple_key_allowed = 1;
1610 
1611  /* Consume the token. */
1612 
1613  start_mark = parser->mark;
1614  SKIP(parser);
1615  end_mark = parser->mark;
1616 
1617  /* Create the BLOCK-ENTRY token and append it to the queue. */
1618 
1619  TOKEN_INIT(token, YAML_BLOCK_ENTRY_TOKEN, start_mark, end_mark);
1620 
1621  if (!ENQUEUE(parser, parser->tokens, token))
1622  return 0;
1623 
1624  return 1;
1625 }
1626 
1627 /*
1628  * Produce the KEY token.
1629  */
1630 
1631 static int
1633 {
1634  yaml_mark_t start_mark, end_mark;
1636 
1637  /* In the block context, additional checks are required. */
1638 
1639  if (!parser->flow_level)
1640  {
1641  /* Check if we are allowed to start a new key (not nessesary simple). */
1642 
1643  if (!parser->simple_key_allowed) {
1644  return yaml_parser_set_scanner_error(parser, NULL, parser->mark,
1645  "mapping keys are not allowed in this context");
1646  }
1647 
1648  /* Add the BLOCK-MAPPING-START token if needed. */
1649 
1650  if (!yaml_parser_roll_indent(parser, parser->mark.column, -1,
1652  return 0;
1653  }
1654 
1655  /* Reset any potential simple keys on the current flow level. */
1656 
1657  if (!yaml_parser_remove_simple_key(parser))
1658  return 0;
1659 
1660  /* Simple keys are allowed after '?' in the block context. */
1661 
1662  parser->simple_key_allowed = (!parser->flow_level);
1663 
1664  /* Consume the token. */
1665 
1666  start_mark = parser->mark;
1667  SKIP(parser);
1668  end_mark = parser->mark;
1669 
1670  /* Create the KEY token and append it to the queue. */
1671 
1672  TOKEN_INIT(token, YAML_KEY_TOKEN, start_mark, end_mark);
1673 
1674  if (!ENQUEUE(parser, parser->tokens, token))
1675  return 0;
1676 
1677  return 1;
1678 }
1679 
1680 /*
1681  * Produce the VALUE token.
1682  */
1683 
1684 static int
1686 {
1687  yaml_mark_t start_mark, end_mark;
1689  yaml_simple_key_t *simple_key = parser->simple_keys.top-1;
1690 
1691  /* Have we found a simple key? */
1692 
1693  if (simple_key->possible)
1694  {
1695 
1696  /* Create the KEY token and insert it into the queue. */
1697 
1698  TOKEN_INIT(token, YAML_KEY_TOKEN, simple_key->mark, simple_key->mark);
1699 
1700  if (!QUEUE_INSERT(parser, parser->tokens,
1701  simple_key->token_number - parser->tokens_parsed, token))
1702  return 0;
1703 
1704  /* In the block context, we may need to add the BLOCK-MAPPING-START token. */
1705 
1706  if (!yaml_parser_roll_indent(parser, simple_key->mark.column,
1707  simple_key->token_number,
1708  YAML_BLOCK_MAPPING_START_TOKEN, simple_key->mark))
1709  return 0;
1710 
1711  /* Remove the simple key. */
1712 
1713  simple_key->possible = 0;
1714 
1715  /* A simple key cannot follow another simple key. */
1716 
1717  parser->simple_key_allowed = 0;
1718  }
1719  else
1720  {
1721  /* The ':' indicator follows a complex key. */
1722 
1723  /* In the block context, extra checks are required. */
1724 
1725  if (!parser->flow_level)
1726  {
1727  /* Check if we are allowed to start a complex value. */
1728 
1729  if (!parser->simple_key_allowed) {
1730  return yaml_parser_set_scanner_error(parser, NULL, parser->mark,
1731  "mapping values are not allowed in this context");
1732  }
1733 
1734  /* Add the BLOCK-MAPPING-START token if needed. */
1735 
1736  if (!yaml_parser_roll_indent(parser, parser->mark.column, -1,
1738  return 0;
1739  }
1740 
1741  /* Simple keys after ':' are allowed in the block context. */
1742 
1743  parser->simple_key_allowed = (!parser->flow_level);
1744  }
1745 
1746  /* Consume the token. */
1747 
1748  start_mark = parser->mark;
1749  SKIP(parser);
1750  end_mark = parser->mark;
1751 
1752  /* Create the VALUE token and append it to the queue. */
1753 
1754  TOKEN_INIT(token, YAML_VALUE_TOKEN, start_mark, end_mark);
1755 
1756  if (!ENQUEUE(parser, parser->tokens, token))
1757  return 0;
1758 
1759  return 1;
1760 }
1761 
1762 /*
1763  * Produce the ALIAS or ANCHOR token.
1764  */
1765 
1766 static int
1768 {
1770 
1771  /* An anchor or an alias could be a simple key. */
1772 
1773  if (!yaml_parser_save_simple_key(parser))
1774  return 0;
1775 
1776  /* A simple key cannot follow an anchor or an alias. */
1777 
1778  parser->simple_key_allowed = 0;
1779 
1780  /* Create the ALIAS or ANCHOR token and append it to the queue. */
1781 
1782  if (!yaml_parser_scan_anchor(parser, &token, type))
1783  return 0;
1784 
1785  if (!ENQUEUE(parser, parser->tokens, token)) {
1786  yaml_token_delete(&token);
1787  return 0;
1788  }
1789  return 1;
1790 }
1791 
1792 /*
1793  * Produce the TAG token.
1794  */
1795 
1796 static int
1798 {
1800 
1801  /* A tag could be a simple key. */
1802 
1803  if (!yaml_parser_save_simple_key(parser))
1804  return 0;
1805 
1806  /* A simple key cannot follow a tag. */
1807 
1808  parser->simple_key_allowed = 0;
1809 
1810  /* Create the TAG token and append it to the queue. */
1811 
1812  if (!yaml_parser_scan_tag(parser, &token))
1813  return 0;
1814 
1815  if (!ENQUEUE(parser, parser->tokens, token)) {
1816  yaml_token_delete(&token);
1817  return 0;
1818  }
1819 
1820  return 1;
1821 }
1822 
1823 /*
1824  * Produce the SCALAR(...,literal) or SCALAR(...,folded) tokens.
1825  */
1826 
1827 static int
1829 {
1831 
1832  /* Remove any potential simple keys. */
1833 
1834  if (!yaml_parser_remove_simple_key(parser))
1835  return 0;
1836 
1837  /* A simple key may follow a block scalar. */
1838 
1839  parser->simple_key_allowed = 1;
1840 
1841  /* Create the SCALAR token and append it to the queue. */
1842 
1843  if (!yaml_parser_scan_block_scalar(parser, &token, literal))
1844  return 0;
1845 
1846  if (!ENQUEUE(parser, parser->tokens, token)) {
1847  yaml_token_delete(&token);
1848  return 0;
1849  }
1850 
1851  return 1;
1852 }
1853 
1854 /*
1855  * Produce the SCALAR(...,single-quoted) or SCALAR(...,double-quoted) tokens.
1856  */
1857 
1858 static int
1860 {
1862 
1863  /* A plain scalar could be a simple key. */
1864 
1865  if (!yaml_parser_save_simple_key(parser))
1866  return 0;
1867 
1868  /* A simple key cannot follow a flow scalar. */
1869 
1870  parser->simple_key_allowed = 0;
1871 
1872  /* Create the SCALAR token and append it to the queue. */
1873 
1874  if (!yaml_parser_scan_flow_scalar(parser, &token, single))
1875  return 0;
1876 
1877  if (!ENQUEUE(parser, parser->tokens, token)) {
1878  yaml_token_delete(&token);
1879  return 0;
1880  }
1881 
1882  return 1;
1883 }
1884 
1885 /*
1886  * Produce the SCALAR(...,plain) token.
1887  */
1888 
1889 static int
1891 {
1893 
1894  /* A plain scalar could be a simple key. */
1895 
1896  if (!yaml_parser_save_simple_key(parser))
1897  return 0;
1898 
1899  /* A simple key cannot follow a flow scalar. */
1900 
1901  parser->simple_key_allowed = 0;
1902 
1903  /* Create the SCALAR token and append it to the queue. */
1904 
1905  if (!yaml_parser_scan_plain_scalar(parser, &token))
1906  return 0;
1907 
1908  if (!ENQUEUE(parser, parser->tokens, token)) {
1909  yaml_token_delete(&token);
1910  return 0;
1911  }
1912 
1913  return 1;
1914 }
1915 
1916 /*
1917  * Eat whitespaces and comments until the next token is found.
1918  */
1919 
1920 static int
1922 {
1923  /* Until the next token is not found. */
1924 
1925  while (1)
1926  {
1927  /* Allow the BOM mark to start a line. */
1928 
1929  if (!CACHE(parser, 1)) return 0;
1930 
1931  if (parser->mark.column == 0 && IS_BOM(parser->buffer))
1932  SKIP(parser);
1933 
1934  /*
1935  * Eat whitespaces.
1936  *
1937  * Tabs are allowed:
1938  *
1939  * - in the flow context;
1940  * - in the block context, but not at the beginning of the line or
1941  * after '-', '?', or ':' (complex value).
1942  */
1943 
1944  if (!CACHE(parser, 1)) return 0;
1945 
1946  while (CHECK(parser->buffer,' ') ||
1947  ((parser->flow_level || !parser->simple_key_allowed) &&
1948  CHECK(parser->buffer, '\t'))) {
1949  SKIP(parser);
1950  if (!CACHE(parser, 1)) return 0;
1951  }
1952 
1953  /* Eat a comment until a line break. */
1954 
1955  if (CHECK(parser->buffer, '#')) {
1956  while (!IS_BREAKZ(parser->buffer)) {
1957  SKIP(parser);
1958  if (!CACHE(parser, 1)) return 0;
1959  }
1960  }
1961 
1962  /* If it is a line break, eat it. */
1963 
1964  if (IS_BREAK(parser->buffer))
1965  {
1966  if (!CACHE(parser, 2)) return 0;
1967  SKIP_LINE(parser);
1968 
1969  /* In the block context, a new line may start a simple key. */
1970 
1971  if (!parser->flow_level) {
1972  parser->simple_key_allowed = 1;
1973  }
1974  }
1975  else
1976  {
1977  /* We have found a token. */
1978 
1979  break;
1980  }
1981  }
1982 
1983  return 1;
1984 }
1985 
1986 /*
1987  * Scan a YAML-DIRECTIVE or TAG-DIRECTIVE token.
1988  *
1989  * Scope:
1990  * %YAML 1.1 # a comment \n
1991  * ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1992  * %TAG !yaml! tag:yaml.org,2002: \n
1993  * ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1994  */
1995 
1996 int
1998 {
1999  yaml_mark_t start_mark, end_mark;
2000  yaml_char_t *name = NULL;
2001  int major, minor;
2002  yaml_char_t *handle = NULL, *prefix = NULL;
2003 
2004  /* Eat '%'. */
2005 
2006  start_mark = parser->mark;
2007 
2008  SKIP(parser);
2009 
2010  /* Scan the directive name. */
2011 
2012  if (!yaml_parser_scan_directive_name(parser, start_mark, &name))
2013  goto error;
2014 
2015  /* Is it a YAML directive? */
2016 
2017  if (strcmp((char *)name, "YAML") == 0)
2018  {
2019  /* Scan the VERSION directive value. */
2020 
2021  if (!yaml_parser_scan_version_directive_value(parser, start_mark,
2022  &major, &minor))
2023  goto error;
2024 
2025  end_mark = parser->mark;
2026 
2027  /* Create a VERSION-DIRECTIVE token. */
2028 
2029  VERSION_DIRECTIVE_TOKEN_INIT(*token, major, minor,
2030  start_mark, end_mark);
2031  }
2032 
2033  /* Is it a TAG directive? */
2034 
2035  else if (strcmp((char *)name, "TAG") == 0)
2036  {
2037  /* Scan the TAG directive value. */
2038 
2039  if (!yaml_parser_scan_tag_directive_value(parser, start_mark,
2040  &handle, &prefix))
2041  goto error;
2042 
2043  end_mark = parser->mark;
2044 
2045  /* Create a TAG-DIRECTIVE token. */
2046 
2047  TAG_DIRECTIVE_TOKEN_INIT(*token, handle, prefix,
2048  start_mark, end_mark);
2049  }
2050 
2051  /* Unknown directive. */
2052 
2053  else
2054  {
2055  yaml_parser_set_scanner_error(parser, "while scanning a directive",
2056  start_mark, "found unknown directive name");
2057  goto error;
2058  }
2059 
2060  /* Eat the rest of the line including any comments. */
2061 
2062  if (!CACHE(parser, 1)) goto error;
2063 
2064  while (IS_BLANK(parser->buffer)) {
2065  SKIP(parser);
2066  if (!CACHE(parser, 1)) goto error;
2067  }
2068 
2069  if (CHECK(parser->buffer, '#')) {
2070  while (!IS_BREAKZ(parser->buffer)) {
2071  SKIP(parser);
2072  if (!CACHE(parser, 1)) goto error;
2073  }
2074  }
2075 
2076  /* Check if we are at the end of the line. */
2077 
2078  if (!IS_BREAKZ(parser->buffer)) {
2079  yaml_parser_set_scanner_error(parser, "while scanning a directive",
2080  start_mark, "did not find expected comment or line break");
2081  goto error;
2082  }
2083 
2084  /* Eat a line break. */
2085 
2086  if (IS_BREAK(parser->buffer)) {
2087  if (!CACHE(parser, 2)) goto error;
2088  SKIP_LINE(parser);
2089  }
2090 
2091  yaml_free(name);
2092 
2093  return 1;
2094 
2095 error:
2096  yaml_free(prefix);
2097  yaml_free(handle);
2098  yaml_free(name);
2099  return 0;
2100 }
2101 
2102 /*
2103  * Scan the directive name.
2104  *
2105  * Scope:
2106  * %YAML 1.1 # a comment \n
2107  * ^^^^
2108  * %TAG !yaml! tag:yaml.org,2002: \n
2109  * ^^^
2110  */
2111 
2112 static int
2114  yaml_mark_t start_mark, yaml_char_t **name)
2115 {
2116  yaml_string_t string = NULL_STRING;
2117 
2118  if (!STRING_INIT(parser, string, INITIAL_STRING_SIZE)) goto error;
2119 
2120  /* Consume the directive name. */
2121 
2122  if (!CACHE(parser, 1)) goto error;
2123 
2124  while (IS_ALPHA(parser->buffer))
2125  {
2126  if (!READ(parser, string)) goto error;
2127  if (!CACHE(parser, 1)) goto error;
2128  }
2129 
2130  /* Check if the name is empty. */
2131 
2132  if (string.start == string.pointer) {
2133  yaml_parser_set_scanner_error(parser, "while scanning a directive",
2134  start_mark, "could not find expected directive name");
2135  goto error;
2136  }
2137 
2138  /* Check for an blank character after the name. */
2139 
2140  if (!IS_BLANKZ(parser->buffer)) {
2141  yaml_parser_set_scanner_error(parser, "while scanning a directive",
2142  start_mark, "found unexpected non-alphabetical character");
2143  goto error;
2144  }
2145 
2146  *name = string.start;
2147 
2148  return 1;
2149 
2150 error:
2151  STRING_DEL(parser, string);
2152  return 0;
2153 }
2154 
2155 /*
2156  * Scan the value of VERSION-DIRECTIVE.
2157  *
2158  * Scope:
2159  * %YAML 1.1 # a comment \n
2160  * ^^^^^^
2161  */
2162 
2163 static int
2165  yaml_mark_t start_mark, int *major, int *minor)
2166 {
2167  /* Eat whitespaces. */
2168 
2169  if (!CACHE(parser, 1)) return 0;
2170 
2171  while (IS_BLANK(parser->buffer)) {
2172  SKIP(parser);
2173  if (!CACHE(parser, 1)) return 0;
2174  }
2175 
2176  /* Consume the major version number. */
2177 
2178  if (!yaml_parser_scan_version_directive_number(parser, start_mark, major))
2179  return 0;
2180 
2181  /* Eat '.'. */
2182 
2183  if (!CHECK(parser->buffer, '.')) {
2184  return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive",
2185  start_mark, "did not find expected digit or '.' character");
2186  }
2187 
2188  SKIP(parser);
2189 
2190  /* Consume the minor version number. */
2191 
2192  if (!yaml_parser_scan_version_directive_number(parser, start_mark, minor))
2193  return 0;
2194 
2195  return 1;
2196 }
2197 
2198 #define MAX_NUMBER_LENGTH 9
2199 
2200 /*
2201  * Scan the version number of VERSION-DIRECTIVE.
2202  *
2203  * Scope:
2204  * %YAML 1.1 # a comment \n
2205  * ^
2206  * %YAML 1.1 # a comment \n
2207  * ^
2208  */
2209 
2210 static int
2212  yaml_mark_t start_mark, int *number)
2213 {
2214  int value = 0;
2215  size_t length = 0;
2216 
2217  /* Repeat while the next character is digit. */
2218 
2219  if (!CACHE(parser, 1)) return 0;
2220 
2221  while (IS_DIGIT(parser->buffer))
2222  {
2223  /* Check if the number is too long. */
2224 
2225  if (++length > MAX_NUMBER_LENGTH) {
2226  return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive",
2227  start_mark, "found extremely long version number");
2228  }
2229 
2230  value = value*10 + AS_DIGIT(parser->buffer);
2231 
2232  SKIP(parser);
2233 
2234  if (!CACHE(parser, 1)) return 0;
2235  }
2236 
2237  /* Check if the number was present. */
2238 
2239  if (!length) {
2240  return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive",
2241  start_mark, "did not find expected version number");
2242  }
2243 
2244  *number = value;
2245 
2246  return 1;
2247 }
2248 
2249 /*
2250  * Scan the value of a TAG-DIRECTIVE token.
2251  *
2252  * Scope:
2253  * %TAG !yaml! tag:yaml.org,2002: \n
2254  * ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2255  */
2256 
2257 static int
2259  yaml_mark_t start_mark, yaml_char_t **handle, yaml_char_t **prefix)
2260 {
2261  yaml_char_t *handle_value = NULL;
2262  yaml_char_t *prefix_value = NULL;
2263 
2264  /* Eat whitespaces. */
2265 
2266  if (!CACHE(parser, 1)) goto error;
2267 
2268  while (IS_BLANK(parser->buffer)) {
2269  SKIP(parser);
2270  if (!CACHE(parser, 1)) goto error;
2271  }
2272 
2273  /* Scan a handle. */
2274 
2275  if (!yaml_parser_scan_tag_handle(parser, 1, start_mark, &handle_value))
2276  goto error;
2277 
2278  /* Expect a whitespace. */
2279 
2280  if (!CACHE(parser, 1)) goto error;
2281 
2282  if (!IS_BLANK(parser->buffer)) {
2283  yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive",
2284  start_mark, "did not find expected whitespace");
2285  goto error;
2286  }
2287 
2288  /* Eat whitespaces. */
2289 
2290  while (IS_BLANK(parser->buffer)) {
2291  SKIP(parser);
2292  if (!CACHE(parser, 1)) goto error;
2293  }
2294 
2295  /* Scan a prefix. */
2296 
2297  if (!yaml_parser_scan_tag_uri(parser, 1, NULL, start_mark, &prefix_value))
2298  goto error;
2299 
2300  /* Expect a whitespace or line break. */
2301 
2302  if (!CACHE(parser, 1)) goto error;
2303 
2304  if (!IS_BLANKZ(parser->buffer)) {
2305  yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive",
2306  start_mark, "did not find expected whitespace or line break");
2307  goto error;
2308  }
2309 
2310  *handle = handle_value;
2311  *prefix = prefix_value;
2312 
2313  return 1;
2314 
2315 error:
2316  yaml_free(handle_value);
2317  yaml_free(prefix_value);
2318  return 0;
2319 }
2320 
2321 static int
2323  yaml_token_type_t type)
2324 {
2325  int length = 0;
2326  yaml_mark_t start_mark, end_mark;
2327  yaml_string_t string = NULL_STRING;
2328 
2329  if (!STRING_INIT(parser, string, INITIAL_STRING_SIZE)) goto error;
2330 
2331  /* Eat the indicator character. */
2332 
2333  start_mark = parser->mark;
2334 
2335  SKIP(parser);
2336 
2337  /* Consume the value. */
2338 
2339  if (!CACHE(parser, 1)) goto error;
2340 
2341  while (IS_ALPHA(parser->buffer)) {
2342  if (!READ(parser, string)) goto error;
2343  if (!CACHE(parser, 1)) goto error;
2344  length ++;
2345  }
2346 
2347  end_mark = parser->mark;
2348 
2349  /*
2350  * Check if length of the anchor is greater than 0 and it is followed by
2351  * a whitespace character or one of the indicators:
2352  *
2353  * '?', ':', ',', ']', '}', '%', '@', '`'.
2354  */
2355 
2356  if (!length || !(IS_BLANKZ(parser->buffer) || CHECK(parser->buffer, '?')
2357  || CHECK(parser->buffer, ':') || CHECK(parser->buffer, ',')
2358  || CHECK(parser->buffer, ']') || CHECK(parser->buffer, '}')
2359  || CHECK(parser->buffer, '%') || CHECK(parser->buffer, '@')
2360  || CHECK(parser->buffer, '`'))) {
2362  "while scanning an anchor" : "while scanning an alias", start_mark,
2363  "did not find expected alphabetic or numeric character");
2364  goto error;
2365  }
2366 
2367  /* Create a token. */
2368 
2369  if (type == YAML_ANCHOR_TOKEN) {
2370  ANCHOR_TOKEN_INIT(*token, string.start, start_mark, end_mark);
2371  }
2372  else {
2373  ALIAS_TOKEN_INIT(*token, string.start, start_mark, end_mark);
2374  }
2375 
2376  return 1;
2377 
2378 error:
2379  STRING_DEL(parser, string);
2380  return 0;
2381 }
2382 
2383 /*
2384  * Scan a TAG token.
2385  */
2386 
2387 static int
2389 {
2390  yaml_char_t *handle = NULL;
2391  yaml_char_t *suffix = NULL;
2392  yaml_mark_t start_mark, end_mark;
2393 
2394  start_mark = parser->mark;
2395 
2396  /* Check if the tag is in the canonical form. */
2397 
2398  if (!CACHE(parser, 2)) goto error;
2399 
2400  if (CHECK_AT(parser->buffer, '<', 1))
2401  {
2402  /* Set the handle to '' */
2403 
2404  handle = yaml_malloc(1);
2405  if (!handle) goto error;
2406  handle[0] = '\0';
2407 
2408  /* Eat '!<' */
2409 
2410  SKIP(parser);
2411  SKIP(parser);
2412 
2413  /* Consume the tag value. */
2414 
2415  if (!yaml_parser_scan_tag_uri(parser, 0, NULL, start_mark, &suffix))
2416  goto error;
2417 
2418  /* Check for '>' and eat it. */
2419 
2420  if (!CHECK(parser->buffer, '>')) {
2421  yaml_parser_set_scanner_error(parser, "while scanning a tag",
2422  start_mark, "did not find the expected '>'");
2423  goto error;
2424  }
2425 
2426  SKIP(parser);
2427  }
2428  else
2429  {
2430  /* The tag has either the '!suffix' or the '!handle!suffix' form. */
2431 
2432  /* First, try to scan a handle. */
2433 
2434  if (!yaml_parser_scan_tag_handle(parser, 0, start_mark, &handle))
2435  goto error;
2436 
2437  /* Check if it is, indeed, handle. */
2438 
2439  if (handle[0] == '!' && handle[1] != '\0' && handle[strlen((char *)handle)-1] == '!')
2440  {
2441  /* Scan the suffix now. */
2442 
2443  if (!yaml_parser_scan_tag_uri(parser, 0, NULL, start_mark, &suffix))
2444  goto error;
2445  }
2446  else
2447  {
2448  /* It wasn't a handle after all. Scan the rest of the tag. */
2449 
2450  if (!yaml_parser_scan_tag_uri(parser, 0, handle, start_mark, &suffix))
2451  goto error;
2452 
2453  /* Set the handle to '!'. */
2454 
2455  yaml_free(handle);
2456  handle = yaml_malloc(2);
2457  if (!handle) goto error;
2458  handle[0] = '!';
2459  handle[1] = '\0';
2460 
2461  /*
2462  * A special case: the '!' tag. Set the handle to '' and the
2463  * suffix to '!'.
2464  */
2465 
2466  if (suffix[0] == '\0') {
2467  yaml_char_t *tmp = handle;
2468  handle = suffix;
2469  suffix = tmp;
2470  }
2471  }
2472  }
2473 
2474  /* Check the character which ends the tag. */
2475 
2476  if (!CACHE(parser, 1)) goto error;
2477 
2478  if (!IS_BLANKZ(parser->buffer)) {
2479  yaml_parser_set_scanner_error(parser, "while scanning a tag",
2480  start_mark, "did not find expected whitespace or line break");
2481  goto error;
2482  }
2483 
2484  end_mark = parser->mark;
2485 
2486  /* Create a token. */
2487 
2488  TAG_TOKEN_INIT(*token, handle, suffix, start_mark, end_mark);
2489 
2490  return 1;
2491 
2492 error:
2493  yaml_free(handle);
2494  yaml_free(suffix);
2495  return 0;
2496 }
2497 
2498 /*
2499  * Scan a tag handle.
2500  */
2501 
2502 static int
2504  yaml_mark_t start_mark, yaml_char_t **handle)
2505 {
2506  yaml_string_t string = NULL_STRING;
2507 
2508  if (!STRING_INIT(parser, string, INITIAL_STRING_SIZE)) goto error;
2509 
2510  /* Check the initial '!' character. */
2511 
2512  if (!CACHE(parser, 1)) goto error;
2513 
2514  if (!CHECK(parser->buffer, '!')) {
2515  yaml_parser_set_scanner_error(parser, directive ?
2516  "while scanning a tag directive" : "while scanning a tag",
2517  start_mark, "did not find expected '!'");
2518  goto error;
2519  }
2520 
2521  /* Copy the '!' character. */
2522 
2523  if (!READ(parser, string)) goto error;
2524 
2525  /* Copy all subsequent alphabetical and numerical characters. */
2526 
2527  if (!CACHE(parser, 1)) goto error;
2528 
2529  while (IS_ALPHA(parser->buffer))
2530  {
2531  if (!READ(parser, string)) goto error;
2532  if (!CACHE(parser, 1)) goto error;
2533  }
2534 
2535  /* Check if the trailing character is '!' and copy it. */
2536 
2537  if (CHECK(parser->buffer, '!'))
2538  {
2539  if (!READ(parser, string)) goto error;
2540  }
2541  else
2542  {
2543  /*
2544  * It's either the '!' tag or not really a tag handle. If it's a %TAG
2545  * directive, it's an error. If it's a tag token, it must be a part of
2546  * URI.
2547  */
2548 
2549  if (directive && !(string.start[0] == '!' && string.start[1] == '\0')) {
2550  yaml_parser_set_scanner_error(parser, "while parsing a tag directive",
2551  start_mark, "did not find expected '!'");
2552  goto error;
2553  }
2554  }
2555 
2556  *handle = string.start;
2557 
2558  return 1;
2559 
2560 error:
2561  STRING_DEL(parser, string);
2562  return 0;
2563 }
2564 
2565 /*
2566  * Scan a tag.
2567  */
2568 
2569 static int
2571  yaml_char_t *head, yaml_mark_t start_mark, yaml_char_t **uri)
2572 {
2573  size_t length = head ? strlen((char *)head) : 0;
2574  yaml_string_t string = NULL_STRING;
2575 
2576  if (!STRING_INIT(parser, string, INITIAL_STRING_SIZE)) goto error;
2577 
2578  /* Resize the string to include the head. */
2579 
2580  while ((size_t)(string.end - string.start) <= length) {
2581  if (!yaml_string_extend(&string.start, &string.pointer, &string.end)) {
2582  parser->error = YAML_MEMORY_ERROR;
2583  goto error;
2584  }
2585  }
2586 
2587  /*
2588  * Copy the head if needed.
2589  *
2590  * Note that we don't copy the leading '!' character.
2591  */
2592 
2593  if (length > 1) {
2594  memcpy(string.start, head+1, length-1);
2595  string.pointer += length-1;
2596  }
2597 
2598  /* Scan the tag. */
2599 
2600  if (!CACHE(parser, 1)) goto error;
2601 
2602  /*
2603  * The set of characters that may appear in URI is as follows:
2604  *
2605  * '0'-'9', 'A'-'Z', 'a'-'z', '_', '-', ';', '/', '?', ':', '@', '&',
2606  * '=', '+', '$', ',', '.', '!', '~', '*', '\'', '(', ')', '[', ']',
2607  * '%'.
2608  */
2609 
2610  while (IS_ALPHA(parser->buffer) || CHECK(parser->buffer, ';')
2611  || CHECK(parser->buffer, '/') || CHECK(parser->buffer, '?')
2612  || CHECK(parser->buffer, ':') || CHECK(parser->buffer, '@')
2613  || CHECK(parser->buffer, '&') || CHECK(parser->buffer, '=')
2614  || CHECK(parser->buffer, '+') || CHECK(parser->buffer, '$')
2615  || CHECK(parser->buffer, ',') || CHECK(parser->buffer, '.')
2616  || CHECK(parser->buffer, '!') || CHECK(parser->buffer, '~')
2617  || CHECK(parser->buffer, '*') || CHECK(parser->buffer, '\'')
2618  || CHECK(parser->buffer, '(') || CHECK(parser->buffer, ')')
2619  || CHECK(parser->buffer, '[') || CHECK(parser->buffer, ']')
2620  || CHECK(parser->buffer, '%'))
2621  {
2622  /* Check if it is a URI-escape sequence. */
2623 
2624  if (CHECK(parser->buffer, '%')) {
2625  if (!STRING_EXTEND(parser, string))
2626  goto error;
2627 
2628  if (!yaml_parser_scan_uri_escapes(parser,
2629  directive, start_mark, &string)) goto error;
2630  }
2631  else {
2632  if (!READ(parser, string)) goto error;
2633  }
2634 
2635  length ++;
2636  if (!CACHE(parser, 1)) goto error;
2637  }
2638 
2639  /* Check if the tag is non-empty. */
2640 
2641  if (!length) {
2642  if (!STRING_EXTEND(parser, string))
2643  goto error;
2644 
2645  yaml_parser_set_scanner_error(parser, directive ?
2646  "while parsing a %TAG directive" : "while parsing a tag",
2647  start_mark, "did not find expected tag URI");
2648  goto error;
2649  }
2650 
2651  *uri = string.start;
2652 
2653  return 1;
2654 
2655 error:
2656  STRING_DEL(parser, string);
2657  return 0;
2658 }
2659 
2660 /*
2661  * Decode an URI-escape sequence corresponding to a single UTF-8 character.
2662  */
2663 
2664 static int
2666  yaml_mark_t start_mark, yaml_string_t *string)
2667 {
2668  int width = 0;
2669 
2670  /* Decode the required number of characters. */
2671 
2672  do {
2673 
2674  unsigned char octet = 0;
2675 
2676  /* Check for a URI-escaped octet. */
2677 
2678  if (!CACHE(parser, 3)) return 0;
2679 
2680  if (!(CHECK(parser->buffer, '%')
2681  && IS_HEX_AT(parser->buffer, 1)
2682  && IS_HEX_AT(parser->buffer, 2))) {
2683  return yaml_parser_set_scanner_error(parser, directive ?
2684  "while parsing a %TAG directive" : "while parsing a tag",
2685  start_mark, "did not find URI escaped octet");
2686  }
2687 
2688  /* Get the octet. */
2689 
2690  octet = (AS_HEX_AT(parser->buffer, 1) << 4) + AS_HEX_AT(parser->buffer, 2);
2691 
2692  /* If it is the leading octet, determine the length of the UTF-8 sequence. */
2693 
2694  if (!width)
2695  {
2696  width = (octet & 0x80) == 0x00 ? 1 :
2697  (octet & 0xE0) == 0xC0 ? 2 :
2698  (octet & 0xF0) == 0xE0 ? 3 :
2699  (octet & 0xF8) == 0xF0 ? 4 : 0;
2700  if (!width) {
2701  return yaml_parser_set_scanner_error(parser, directive ?
2702  "while parsing a %TAG directive" : "while parsing a tag",
2703  start_mark, "found an incorrect leading UTF-8 octet");
2704  }
2705  }
2706  else
2707  {
2708  /* Check if the trailing octet is correct. */
2709 
2710  if ((octet & 0xC0) != 0x80) {
2711  return yaml_parser_set_scanner_error(parser, directive ?
2712  "while parsing a %TAG directive" : "while parsing a tag",
2713  start_mark, "found an incorrect trailing UTF-8 octet");
2714  }
2715  }
2716 
2717  /* Copy the octet and move the pointers. */
2718 
2719  *(string->pointer++) = octet;
2720  SKIP(parser);
2721  SKIP(parser);
2722  SKIP(parser);
2723 
2724  } while (--width);
2725 
2726  return 1;
2727 }
2728 
2729 /*
2730  * Scan a block scalar.
2731  */
2732 
2733 static int
2735  int literal)
2736 {
2737  yaml_mark_t start_mark;
2738  yaml_mark_t end_mark;
2739  yaml_string_t string = NULL_STRING;
2740  yaml_string_t leading_break = NULL_STRING;
2741  yaml_string_t trailing_breaks = NULL_STRING;
2742  int chomping = 0;
2743  int increment = 0;
2744  int indent = 0;
2745  int leading_blank = 0;
2746  int trailing_blank = 0;
2747 
2748  if (!STRING_INIT(parser, string, INITIAL_STRING_SIZE)) goto error;
2749  if (!STRING_INIT(parser, leading_break, INITIAL_STRING_SIZE)) goto error;
2750  if (!STRING_INIT(parser, trailing_breaks, INITIAL_STRING_SIZE)) goto error;
2751 
2752  /* Eat the indicator '|' or '>'. */
2753 
2754  start_mark = parser->mark;
2755 
2756  SKIP(parser);
2757 
2758  /* Scan the additional block scalar indicators. */
2759 
2760  if (!CACHE(parser, 1)) goto error;
2761 
2762  /* Check for a chomping indicator. */
2763 
2764  if (CHECK(parser->buffer, '+') || CHECK(parser->buffer, '-'))
2765  {
2766  /* Set the chomping method and eat the indicator. */
2767 
2768  chomping = CHECK(parser->buffer, '+') ? +1 : -1;
2769 
2770  SKIP(parser);
2771 
2772  /* Check for an indentation indicator. */
2773 
2774  if (!CACHE(parser, 1)) goto error;
2775 
2776  if (IS_DIGIT(parser->buffer))
2777  {
2778  /* Check that the indentation is greater than 0. */
2779 
2780  if (CHECK(parser->buffer, '0')) {
2781  yaml_parser_set_scanner_error(parser, "while scanning a block scalar",
2782  start_mark, "found an indentation indicator equal to 0");
2783  goto error;
2784  }
2785 
2786  /* Get the indentation level and eat the indicator. */
2787 
2788  increment = AS_DIGIT(parser->buffer);
2789 
2790  SKIP(parser);
2791  }
2792  }
2793 
2794  /* Do the same as above, but in the opposite order. */
2795 
2796  else if (IS_DIGIT(parser->buffer))
2797  {
2798  if (CHECK(parser->buffer, '0')) {
2799  yaml_parser_set_scanner_error(parser, "while scanning a block scalar",
2800  start_mark, "found an indentation indicator equal to 0");
2801  goto error;
2802  }
2803 
2804  increment = AS_DIGIT(parser->buffer);
2805 
2806  SKIP(parser);
2807 
2808  if (!CACHE(parser, 1)) goto error;
2809 
2810  if (CHECK(parser->buffer, '+') || CHECK(parser->buffer, '-')) {
2811  chomping = CHECK(parser->buffer, '+') ? +1 : -1;
2812 
2813  SKIP(parser);
2814  }
2815  }
2816 
2817  /* Eat whitespaces and comments to the end of the line. */
2818 
2819  if (!CACHE(parser, 1)) goto error;
2820 
2821  while (IS_BLANK(parser->buffer)) {
2822  SKIP(parser);
2823  if (!CACHE(parser, 1)) goto error;
2824  }
2825 
2826  if (CHECK(parser->buffer, '#')) {
2827  while (!IS_BREAKZ(parser->buffer)) {
2828  SKIP(parser);
2829  if (!CACHE(parser, 1)) goto error;
2830  }
2831  }
2832 
2833  /* Check if we are at the end of the line. */
2834 
2835  if (!IS_BREAKZ(parser->buffer)) {
2836  yaml_parser_set_scanner_error(parser, "while scanning a block scalar",
2837  start_mark, "did not find expected comment or line break");
2838  goto error;
2839  }
2840 
2841  /* Eat a line break. */
2842 
2843  if (IS_BREAK(parser->buffer)) {
2844  if (!CACHE(parser, 2)) goto error;
2845  SKIP_LINE(parser);
2846  }
2847 
2848  end_mark = parser->mark;
2849 
2850  /* Set the indentation level if it was specified. */
2851 
2852  if (increment) {
2853  indent = parser->indent >= 0 ? parser->indent+increment : increment;
2854  }
2855 
2856  /* Scan the leading line breaks and determine the indentation level if needed. */
2857 
2858  if (!yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks,
2859  start_mark, &end_mark)) goto error;
2860 
2861  /* Scan the block scalar content. */
2862 
2863  if (!CACHE(parser, 1)) goto error;
2864 
2865  while ((int)parser->mark.column == indent && !IS_Z(parser->buffer))
2866  {
2867  /*
2868  * We are at the beginning of a non-empty line.
2869  */
2870 
2871  /* Is it a trailing whitespace? */
2872 
2873  trailing_blank = IS_BLANK(parser->buffer);
2874 
2875  /* Check if we need to fold the leading line break. */
2876 
2877  if (!literal && (*leading_break.start == '\n')
2878  && !leading_blank && !trailing_blank)
2879  {
2880  /* Do we need to join the lines by space? */
2881 
2882  if (*trailing_breaks.start == '\0') {
2883  if (!STRING_EXTEND(parser, string)) goto error;
2884  *(string.pointer ++) = ' ';
2885  }
2886 
2887  CLEAR(parser, leading_break);
2888  }
2889  else {
2890  if (!JOIN(parser, string, leading_break)) goto error;
2891  CLEAR(parser, leading_break);
2892  }
2893 
2894  /* Append the remaining line breaks. */
2895 
2896  if (!JOIN(parser, string, trailing_breaks)) goto error;
2897  CLEAR(parser, trailing_breaks);
2898 
2899  /* Is it a leading whitespace? */
2900 
2901  leading_blank = IS_BLANK(parser->buffer);
2902 
2903  /* Consume the current line. */
2904 
2905  while (!IS_BREAKZ(parser->buffer)) {
2906  if (!READ(parser, string)) goto error;
2907  if (!CACHE(parser, 1)) goto error;
2908  }
2909 
2910  /* Consume the line break. */
2911 
2912  if (!CACHE(parser, 2)) goto error;
2913 
2914  if (!READ_LINE(parser, leading_break)) goto error;
2915 
2916  /* Eat the following indentation spaces and line breaks. */
2917 
2919  &indent, &trailing_breaks, start_mark, &end_mark)) goto error;
2920  }
2921 
2922  /* Chomp the tail. */
2923 
2924  if (chomping != -1) {
2925  if (!JOIN(parser, string, leading_break)) goto error;
2926  }
2927  if (chomping == 1) {
2928  if (!JOIN(parser, string, trailing_breaks)) goto error;
2929  }
2930 
2931  /* Create a token. */
2932 
2933  SCALAR_TOKEN_INIT(*token, string.start, string.pointer-string.start,
2935  start_mark, end_mark);
2936 
2937  STRING_DEL(parser, leading_break);
2938  STRING_DEL(parser, trailing_breaks);
2939 
2940  return 1;
2941 
2942 error:
2943  STRING_DEL(parser, string);
2944  STRING_DEL(parser, leading_break);
2945  STRING_DEL(parser, trailing_breaks);
2946 
2947  return 0;
2948 }
2949 
2950 /*
2951  * Scan indentation spaces and line breaks for a block scalar. Determine the
2952  * indentation level if needed.
2953  */
2954 
2955 static int
2957  int *indent, yaml_string_t *breaks,
2958  yaml_mark_t start_mark, yaml_mark_t *end_mark)
2959 {
2960  int max_indent = 0;
2961 
2962  *end_mark = parser->mark;
2963 
2964  /* Eat the indentation spaces and line breaks. */
2965 
2966  while (1)
2967  {
2968  /* Eat the indentation spaces. */
2969 
2970  if (!CACHE(parser, 1)) return 0;
2971 
2972  while ((!*indent || (int)parser->mark.column < *indent)
2973  && IS_SPACE(parser->buffer)) {
2974  SKIP(parser);
2975  if (!CACHE(parser, 1)) return 0;
2976  }
2977 
2978  if ((int)parser->mark.column > max_indent)
2979  max_indent = (int)parser->mark.column;
2980 
2981  /* Check for a tab character messing the indentation. */
2982 
2983  if ((!*indent || (int)parser->mark.column < *indent)
2984  && IS_TAB(parser->buffer)) {
2985  return yaml_parser_set_scanner_error(parser, "while scanning a block scalar",
2986  start_mark, "found a tab character where an indentation space is expected");
2987  }
2988 
2989  /* Have we found a non-empty line? */
2990 
2991  if (!IS_BREAK(parser->buffer)) break;
2992 
2993  /* Consume the line break. */
2994 
2995  if (!CACHE(parser, 2)) return 0;
2996  if (!READ_LINE(parser, *breaks)) return 0;
2997  *end_mark = parser->mark;
2998  }
2999 
3000  /* Determine the indentation level if needed. */
3001 
3002  if (!*indent) {
3003  *indent = max_indent;
3004  if (*indent < parser->indent + 1)
3005  *indent = parser->indent + 1;
3006  if (*indent < 1)
3007  *indent = 1;
3008  }
3009 
3010  return 1;
3011 }
3012 
3013 /*
3014  * Scan a quoted scalar.
3015  */
3016 
3017 static int
3019  int single)
3020 {
3021  yaml_mark_t start_mark;
3022  yaml_mark_t end_mark;
3023  yaml_string_t string = NULL_STRING;
3024  yaml_string_t leading_break = NULL_STRING;
3025  yaml_string_t trailing_breaks = NULL_STRING;
3026  yaml_string_t whitespaces = NULL_STRING;
3027  int leading_blanks;
3028 
3029  if (!STRING_INIT(parser, string, INITIAL_STRING_SIZE)) goto error;
3030  if (!STRING_INIT(parser, leading_break, INITIAL_STRING_SIZE)) goto error;
3031  if (!STRING_INIT(parser, trailing_breaks, INITIAL_STRING_SIZE)) goto error;
3032  if (!STRING_INIT(parser, whitespaces, INITIAL_STRING_SIZE)) goto error;
3033 
3034  /* Eat the left quote. */
3035 
3036  start_mark = parser->mark;
3037 
3038  SKIP(parser);
3039 
3040  /* Consume the content of the quoted scalar. */
3041 
3042  while (1)
3043  {
3044  /* Check that there are no document indicators at the beginning of the line. */
3045 
3046  if (!CACHE(parser, 4)) goto error;
3047 
3048  if (parser->mark.column == 0 &&
3049  ((CHECK_AT(parser->buffer, '-', 0) &&
3050  CHECK_AT(parser->buffer, '-', 1) &&
3051  CHECK_AT(parser->buffer, '-', 2)) ||
3052  (CHECK_AT(parser->buffer, '.', 0) &&
3053  CHECK_AT(parser->buffer, '.', 1) &&
3054  CHECK_AT(parser->buffer, '.', 2))) &&
3055  IS_BLANKZ_AT(parser->buffer, 3))
3056  {
3057  yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar",
3058  start_mark, "found unexpected document indicator");
3059  goto error;
3060  }
3061 
3062  /* Check for EOF. */
3063 
3064  if (IS_Z(parser->buffer)) {
3065  yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar",
3066  start_mark, "found unexpected end of stream");
3067  goto error;
3068  }
3069 
3070  /* Consume non-blank characters. */
3071 
3072  if (!CACHE(parser, 2)) goto error;
3073 
3074  leading_blanks = 0;
3075 
3076  while (!IS_BLANKZ(parser->buffer))
3077  {
3078  /* Check for an escaped single quote. */
3079 
3080  if (single && CHECK_AT(parser->buffer, '\'', 0)
3081  && CHECK_AT(parser->buffer, '\'', 1))
3082  {
3083  if (!STRING_EXTEND(parser, string)) goto error;
3084  *(string.pointer++) = '\'';
3085  SKIP(parser);
3086  SKIP(parser);
3087  }
3088 
3089  /* Check for the right quote. */
3090 
3091  else if (CHECK(parser->buffer, single ? '\'' : '"'))
3092  {
3093  break;
3094  }
3095 
3096  /* Check for an escaped line break. */
3097 
3098  else if (!single && CHECK(parser->buffer, '\\')
3099  && IS_BREAK_AT(parser->buffer, 1))
3100  {
3101  if (!CACHE(parser, 3)) goto error;
3102  SKIP(parser);
3103  SKIP_LINE(parser);
3104  leading_blanks = 1;
3105  break;
3106  }
3107 
3108  /* Check for an escape sequence. */
3109 
3110  else if (!single && CHECK(parser->buffer, '\\'))
3111  {
3112  size_t code_length = 0;
3113 
3114  if (!STRING_EXTEND(parser, string)) goto error;
3115 
3116  /* Check the escape character. */
3117 
3118  switch (parser->buffer.pointer[1])
3119  {
3120  case '0':
3121  *(string.pointer++) = '\0';
3122  break;
3123 
3124  case 'a':
3125  *(string.pointer++) = '\x07';
3126  break;
3127 
3128  case 'b':
3129  *(string.pointer++) = '\x08';
3130  break;
3131 
3132  case 't':
3133  case '\t':
3134  *(string.pointer++) = '\x09';
3135  break;
3136 
3137  case 'n':
3138  *(string.pointer++) = '\x0A';
3139  break;
3140 
3141  case 'v':
3142  *(string.pointer++) = '\x0B';
3143  break;
3144 
3145  case 'f':
3146  *(string.pointer++) = '\x0C';
3147  break;
3148 
3149  case 'r':
3150  *(string.pointer++) = '\x0D';
3151  break;
3152 
3153  case 'e':
3154  *(string.pointer++) = '\x1B';
3155  break;
3156 
3157  case ' ':
3158  *(string.pointer++) = '\x20';
3159  break;
3160 
3161  case '"':
3162  *(string.pointer++) = '"';
3163  break;
3164 
3165  case '\'':
3166  *(string.pointer++) = '\'';
3167  break;
3168 
3169  case '\\':
3170  *(string.pointer++) = '\\';
3171  break;
3172 
3173  case 'N': /* NEL (#x85) */
3174  *(string.pointer++) = '\xC2';
3175  *(string.pointer++) = '\x85';
3176  break;
3177 
3178  case '_': /* #xA0 */
3179  *(string.pointer++) = '\xC2';
3180  *(string.pointer++) = '\xA0';
3181  break;
3182 
3183  case 'L': /* LS (#x2028) */
3184  *(string.pointer++) = '\xE2';
3185  *(string.pointer++) = '\x80';
3186  *(string.pointer++) = '\xA8';
3187  break;
3188 
3189  case 'P': /* PS (#x2029) */
3190  *(string.pointer++) = '\xE2';
3191  *(string.pointer++) = '\x80';
3192  *(string.pointer++) = '\xA9';
3193  break;
3194 
3195  case 'x':
3196  code_length = 2;
3197  break;
3198 
3199  case 'u':
3200  code_length = 4;
3201  break;
3202 
3203  case 'U':
3204  code_length = 8;
3205  break;
3206 
3207  default:
3208  yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar",
3209  start_mark, "found unknown escape character");
3210  goto error;
3211  }
3212 
3213  SKIP(parser);
3214  SKIP(parser);
3215 
3216  /* Consume an arbitrary escape code. */
3217 
3218  if (code_length)
3219  {
3220  unsigned int value = 0;
3221  size_t k;
3222 
3223  /* Scan the character value. */
3224 
3225  if (!CACHE(parser, code_length)) goto error;
3226 
3227  for (k = 0; k < code_length; k ++) {
3228  if (!IS_HEX_AT(parser->buffer, k)) {
3229  yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar",
3230  start_mark, "did not find expected hexdecimal number");
3231  goto error;
3232  }
3233  value = (value << 4) + AS_HEX_AT(parser->buffer, k);
3234  }
3235 
3236  /* Check the value and write the character. */
3237 
3238  if ((value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF) {
3239  yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar",
3240  start_mark, "found invalid Unicode character escape code");
3241  goto error;
3242  }
3243 
3244  if (value <= 0x7F) {
3245  *(string.pointer++) = value;
3246  }
3247  else if (value <= 0x7FF) {
3248  *(string.pointer++) = 0xC0 + (value >> 6);
3249  *(string.pointer++) = 0x80 + (value & 0x3F);
3250  }
3251  else if (value <= 0xFFFF) {
3252  *(string.pointer++) = 0xE0 + (value >> 12);
3253  *(string.pointer++) = 0x80 + ((value >> 6) & 0x3F);
3254  *(string.pointer++) = 0x80 + (value & 0x3F);
3255  }
3256  else {
3257  *(string.pointer++) = 0xF0 + (value >> 18);
3258  *(string.pointer++) = 0x80 + ((value >> 12) & 0x3F);
3259  *(string.pointer++) = 0x80 + ((value >> 6) & 0x3F);
3260  *(string.pointer++) = 0x80 + (value & 0x3F);
3261  }
3262 
3263  /* Advance the pointer. */
3264 
3265  for (k = 0; k < code_length; k ++) {
3266  SKIP(parser);
3267  }
3268  }
3269  }
3270 
3271  else
3272  {
3273  /* It is a non-escaped non-blank character. */
3274 
3275  if (!READ(parser, string)) goto error;
3276  }
3277 
3278  if (!CACHE(parser, 2)) goto error;
3279  }
3280 
3281  /* Check if we are at the end of the scalar. */
3282 
3283  if (CHECK(parser->buffer, single ? '\'' : '"'))
3284  break;
3285 
3286  /* Consume blank characters. */
3287 
3288  if (!CACHE(parser, 1)) goto error;
3289 
3290  while (IS_BLANK(parser->buffer) || IS_BREAK(parser->buffer))
3291  {
3292  if (IS_BLANK(parser->buffer))
3293  {
3294  /* Consume a space or a tab character. */
3295 
3296  if (!leading_blanks) {
3297  if (!READ(parser, whitespaces)) goto error;
3298  }
3299  else {
3300  SKIP(parser);
3301  }
3302  }
3303  else
3304  {
3305  if (!CACHE(parser, 2)) goto error;
3306 
3307  /* Check if it is a first line break. */
3308 
3309  if (!leading_blanks)
3310  {
3311  CLEAR(parser, whitespaces);
3312  if (!READ_LINE(parser, leading_break)) goto error;
3313  leading_blanks = 1;
3314  }
3315  else
3316  {
3317  if (!READ_LINE(parser, trailing_breaks)) goto error;
3318  }
3319  }
3320  if (!CACHE(parser, 1)) goto error;
3321  }
3322 
3323  /* Join the whitespaces or fold line breaks. */
3324 
3325  if (leading_blanks)
3326  {
3327  /* Do we need to fold line breaks? */
3328 
3329  if (leading_break.start[0] == '\n') {
3330  if (trailing_breaks.start[0] == '\0') {
3331  if (!STRING_EXTEND(parser, string)) goto error;
3332  *(string.pointer++) = ' ';
3333  }
3334  else {
3335  if (!JOIN(parser, string, trailing_breaks)) goto error;
3336  CLEAR(parser, trailing_breaks);
3337  }
3338  CLEAR(parser, leading_break);
3339  }
3340  else {
3341  if (!JOIN(parser, string, leading_break)) goto error;
3342  if (!JOIN(parser, string, trailing_breaks)) goto error;
3343  CLEAR(parser, leading_break);
3344  CLEAR(parser, trailing_breaks);
3345  }
3346  }
3347  else
3348  {
3349  if (!JOIN(parser, string, whitespaces)) goto error;
3350  CLEAR(parser, whitespaces);
3351  }
3352  }
3353 
3354  /* Eat the right quote. */
3355 
3356  SKIP(parser);
3357 
3358  end_mark = parser->mark;
3359 
3360  /* Create a token. */
3361 
3362  SCALAR_TOKEN_INIT(*token, string.start, string.pointer-string.start,
3364  start_mark, end_mark);
3365 
3366  STRING_DEL(parser, leading_break);
3367  STRING_DEL(parser, trailing_breaks);
3368  STRING_DEL(parser, whitespaces);
3369 
3370  return 1;
3371 
3372 error:
3373  STRING_DEL(parser, string);
3374  STRING_DEL(parser, leading_break);
3375  STRING_DEL(parser, trailing_breaks);
3376  STRING_DEL(parser, whitespaces);
3377 
3378  return 0;
3379 }
3380 
3381 /*
3382  * Scan a plain scalar.
3383  */
3384 
3385 static int
3387 {
3388  yaml_mark_t start_mark;
3389  yaml_mark_t end_mark;
3390  yaml_string_t string = NULL_STRING;
3391  yaml_string_t leading_break = NULL_STRING;
3392  yaml_string_t trailing_breaks = NULL_STRING;
3393  yaml_string_t whitespaces = NULL_STRING;
3394  int leading_blanks = 0;
3395  int indent = parser->indent+1;
3396 
3397  if (!STRING_INIT(parser, string, INITIAL_STRING_SIZE)) goto error;
3398  if (!STRING_INIT(parser, leading_break, INITIAL_STRING_SIZE)) goto error;
3399  if (!STRING_INIT(parser, trailing_breaks, INITIAL_STRING_SIZE)) goto error;
3400  if (!STRING_INIT(parser, whitespaces, INITIAL_STRING_SIZE)) goto error;
3401 
3402  start_mark = end_mark = parser->mark;
3403 
3404  /* Consume the content of the plain scalar. */
3405 
3406  while (1)
3407  {
3408  /* Check for a document indicator. */
3409 
3410  if (!CACHE(parser, 4)) goto error;
3411 
3412  if (parser->mark.column == 0 &&
3413  ((CHECK_AT(parser->buffer, '-', 0) &&
3414  CHECK_AT(parser->buffer, '-', 1) &&
3415  CHECK_AT(parser->buffer, '-', 2)) ||
3416  (CHECK_AT(parser->buffer, '.', 0) &&
3417  CHECK_AT(parser->buffer, '.', 1) &&
3418  CHECK_AT(parser->buffer, '.', 2))) &&
3419  IS_BLANKZ_AT(parser->buffer, 3)) break;
3420 
3421  /* Check for a comment. */
3422 
3423  if (CHECK(parser->buffer, '#'))
3424  break;
3425 
3426  /* Consume non-blank characters. */
3427 
3428  while (!IS_BLANKZ(parser->buffer))
3429  {
3430  /* Check for 'x:x' in the flow context. TODO: Fix the test "spec-08-13". */
3431 
3432  if (parser->flow_level
3433  && CHECK(parser->buffer, ':')
3434  && !IS_BLANKZ_AT(parser->buffer, 1)) {
3435  yaml_parser_set_scanner_error(parser, "while scanning a plain scalar",
3436  start_mark, "found unexpected ':'");
3437  goto error;
3438  }
3439 
3440  /* Check for indicators that may end a plain scalar. */
3441 
3442  if ((CHECK(parser->buffer, ':') && IS_BLANKZ_AT(parser->buffer, 1))
3443  || (parser->flow_level &&
3444  (CHECK(parser->buffer, ',') || CHECK(parser->buffer, ':')
3445  || CHECK(parser->buffer, '?') || CHECK(parser->buffer, '[')
3446  || CHECK(parser->buffer, ']') || CHECK(parser->buffer, '{')
3447  || CHECK(parser->buffer, '}'))))
3448  break;
3449 
3450  /* Check if we need to join whitespaces and breaks. */
3451 
3452  if (leading_blanks || whitespaces.start != whitespaces.pointer)
3453  {
3454  if (leading_blanks)
3455  {
3456  /* Do we need to fold line breaks? */
3457 
3458  if (leading_break.start[0] == '\n') {
3459  if (trailing_breaks.start[0] == '\0') {
3460  if (!STRING_EXTEND(parser, string)) goto error;
3461  *(string.pointer++) = ' ';
3462  }
3463  else {
3464  if (!JOIN(parser, string, trailing_breaks)) goto error;
3465  CLEAR(parser, trailing_breaks);
3466  }
3467  CLEAR(parser, leading_break);
3468  }
3469  else {
3470  if (!JOIN(parser, string, leading_break)) goto error;
3471  if (!JOIN(parser, string, trailing_breaks)) goto error;
3472  CLEAR(parser, leading_break);
3473  CLEAR(parser, trailing_breaks);
3474  }
3475 
3476  leading_blanks = 0;
3477  }
3478  else
3479  {
3480  if (!JOIN(parser, string, whitespaces)) goto error;
3481  CLEAR(parser, whitespaces);
3482  }
3483  }
3484 
3485  /* Copy the character. */
3486 
3487  if (!READ(parser, string)) goto error;
3488 
3489  end_mark = parser->mark;
3490 
3491  if (!CACHE(parser, 2)) goto error;
3492  }
3493 
3494  /* Is it the end? */
3495 
3496  if (!(IS_BLANK(parser->buffer) || IS_BREAK(parser->buffer)))
3497  break;
3498 
3499  /* Consume blank characters. */
3500 
3501  if (!CACHE(parser, 1)) goto error;
3502 
3503  while (IS_BLANK(parser->buffer) || IS_BREAK(parser->buffer))
3504  {
3505  if (IS_BLANK(parser->buffer))
3506  {
3507  /* Check for tab character that abuse indentation. */
3508 
3509  if (leading_blanks && (int)parser->mark.column < indent
3510  && IS_TAB(parser->buffer)) {
3511  yaml_parser_set_scanner_error(parser, "while scanning a plain scalar",
3512  start_mark, "found a tab character that violate indentation");
3513  goto error;
3514  }
3515 
3516  /* Consume a space or a tab character. */
3517 
3518  if (!leading_blanks) {
3519  if (!READ(parser, whitespaces)) goto error;
3520  }
3521  else {
3522  SKIP(parser);
3523  }
3524  }
3525  else
3526  {
3527  if (!CACHE(parser, 2)) goto error;
3528 
3529  /* Check if it is a first line break. */
3530 
3531  if (!leading_blanks)
3532  {
3533  CLEAR(parser, whitespaces);
3534  if (!READ_LINE(parser, leading_break)) goto error;
3535  leading_blanks = 1;
3536  }
3537  else
3538  {
3539  if (!READ_LINE(parser, trailing_breaks)) goto error;
3540  }
3541  }
3542  if (!CACHE(parser, 1)) goto error;
3543  }
3544 
3545  /* Check indentation level. */
3546 
3547  if (!parser->flow_level && (int)parser->mark.column < indent)
3548  break;
3549  }
3550 
3551  /* Create a token. */
3552 
3553  SCALAR_TOKEN_INIT(*token, string.start, string.pointer-string.start,
3554  YAML_PLAIN_SCALAR_STYLE, start_mark, end_mark);
3555 
3556  /* Note that we change the 'simple_key_allowed' flag. */
3557 
3558  if (leading_blanks) {
3559  parser->simple_key_allowed = 1;
3560  }
3561 
3562  STRING_DEL(parser, leading_break);
3563  STRING_DEL(parser, trailing_breaks);
3564  STRING_DEL(parser, whitespaces);
3565 
3566  return 1;
3567 
3568 error:
3569  STRING_DEL(parser, string);
3570  STRING_DEL(parser, leading_break);
3571  STRING_DEL(parser, trailing_breaks);
3572  STRING_DEL(parser, whitespaces);
3573 
3574  return 0;
3575 }
3576 
The double-quoted scalar style.
Definition: yaml.h:174
struct yaml_parser_s::@78 tokens
The tokens queue.
#define PUSH(x)
Definition: bigdecimal.c:69
A BLOCK-SEQUENCE-START token.
Definition: yaml.h:232
The pointer position.
Definition: yaml.h:145
static int yaml_parser_scan_directive(yaml_parser_t *parser, yaml_token_t *token)
Definition: scanner.c:1997
static int yaml_parser_fetch_directive(yaml_parser_t *parser)
Definition: scanner.c:1369
size_t strlen(const char *)
yaml_token_t * tail
The tail of the tokens queue.
Definition: yaml.h:1200
A FLOW-SEQUENCE-START token.
Definition: yaml.h:239
#define NULL_STRING
Definition: yaml_private.h:126
A VALUE token.
Definition: yaml.h:254
#define IS_BOM(string)
Definition: yaml_private.h:301
static int yaml_parser_scan_flow_scalar(yaml_parser_t *parser, yaml_token_t *token, int single)
Definition: scanner.c:3018
Cannot allocate or reallocate a block of memory.
Definition: yaml.h:127
static int yaml_parser_scan_tag(yaml_parser_t *parser, yaml_token_t *token)
Definition: scanner.c:2388
#define ALIAS_TOKEN_INIT(token, token_value, start_mark, end_mark)
Definition: yaml_private.h:515
static int yaml_parser_fetch_stream_start(yaml_parser_t *parser)
Definition: scanner.c:1293
A BLOCK-END token.
Definition: yaml.h:236
yaml_string_extend(yaml_char_t **start, yaml_char_t **pointer, yaml_char_t **end)
Definition: api.c:74
struct yaml_parser_s::@79 indents
The indentation levels stack.
#define AS_DIGIT(string)
Definition: yaml_private.h:220
int stream_start_produced
Have we started to scan the input stream?
Definition: yaml.h:1183
static int yaml_parser_fetch_block_scalar(yaml_parser_t *parser, int literal)
Definition: scanner.c:1828
static int yaml_parser_fetch_block_entry(yaml_parser_t *parser)
Definition: scanner.c:1571
yaml_encoding_t encoding
The input encoding.
Definition: yaml.h:1165
The parser structure.
Definition: yaml.h:1081
A BLOCK-SEQUENCE-END token.
Definition: yaml.h:234
if(len<=MAX_WORD_LENGTH &&len >=MIN_WORD_LENGTH)
Definition: zonetab.h:883
#define SKIP(parser)
Definition: scanner.c:494
A FLOW-ENTRY token.
Definition: yaml.h:250
#define READ(parser, string)
Definition: scanner.c:518
static int yaml_parser_fetch_flow_collection_end(yaml_parser_t *parser, yaml_token_type_t type)
Definition: scanner.c:1493
static int yaml_parser_scan_tag_directive_value(yaml_parser_t *parser, yaml_mark_t mark, yaml_char_t **handle, yaml_char_t **prefix)
Definition: scanner.c:2258
static int yaml_parser_scan_block_scalar(yaml_parser_t *parser, yaml_token_t *token, int literal)
Definition: scanner.c:2734
unsigned char yaml_char_t
The character type (UTF-8 octet).
Definition: yaml.h:78
const unsigned char * start
The string start pointer.
Definition: yaml.h:1123
unsigned short token
Definition: id.c:36
const char * context
The error context.
Definition: yaml.h:1099
yaml_mark_t mark
The position mark.
Definition: yaml.h:1004
#define IS_BREAK_AT(string, offset)
Definition: yaml_private.h:332
yaml_char_t * pointer
Definition: yaml_private.h:114
static int yaml_parser_fetch_flow_scalar(yaml_parser_t *parser, int single)
Definition: scanner.c:1859
#define READ_LINE(parser, string)
Definition: scanner.c:530
#define assert(x)
Definition: dlmalloc.c:1176
int indent
The current indentation level.
Definition: yaml.h:1220
#define IS_BLANK(string)
Definition: yaml_private.h:326
static int yaml_parser_scan_version_directive_value(yaml_parser_t *parser, yaml_mark_t start_mark, int *major, int *minor)
Definition: scanner.c:2164
#define TAG_DIRECTIVE_TOKEN_INIT(token, token_handle, token_prefix, start_mark, end_mark)
Definition: yaml_private.h:539
static int yaml_parser_fetch_flow_collection_start(yaml_parser_t *parser, yaml_token_type_t type)
Definition: scanner.c:1450
yaml_char_t * pointer
The current position of the buffer.
Definition: yaml.h:1144
An ALIAS token.
Definition: yaml.h:257
yaml_mark_t mark
The mark of the current position.
Definition: yaml.h:1171
#define IS_HEX_AT(string, offset)
Definition: yaml_private.h:226
yaml_char_t * start
Definition: yaml_private.h:112
static int yaml_parser_remove_simple_key(yaml_parser_t *parser)
Definition: scanner.c:1134
#define DEQUEUE(context, queue)
Definition: yaml_private.h:483
A FLOW-SEQUENCE-END token.
Definition: yaml.h:241
#define INITIAL_STRING_SIZE
Definition: yaml_private.h:89
#define STRING_DEL(context, string)
Definition: yaml_private.h:144
#define IS_SPACE(string)
Definition: yaml_private.h:309
yaml_free(void *ptr)
Definition: api.c:51
The folded scalar style.
Definition: yaml.h:179
#define AS_HEX_AT(string, offset)
Definition: yaml_private.h:240
int * top
The top of the stack.
Definition: yaml.h:1216
#define QUEUE_INSERT(context, queue, index, value)
Definition: yaml_private.h:486
static int yaml_parser_fetch_next_token(yaml_parser_t *parser)
Definition: scanner.c:860
int required
Is a simple key required?
Definition: yaml.h:998
int simple_key_allowed
May a simple key occur at the current position?
Definition: yaml.h:1223
#define CACHE(parser, length)
Definition: scanner.c:485
#define IS_TAB(string)
Definition: yaml_private.h:317
static int yaml_parser_fetch_document_indicator(yaml_parser_t *parser, yaml_token_type_t type)
Definition: scanner.c:1405
static int yaml_parser_scan_uri_escapes(yaml_parser_t *parser, int directive, yaml_mark_t start_mark, yaml_string_t *string)
Definition: scanner.c:2665
static int yaml_parser_unroll_indent(yaml_parser_t *parser, ptrdiff_t column)
Definition: scanner.c:1260
static int yaml_parser_fetch_stream_end(yaml_parser_t *parser)
Definition: scanner.c:1331
static int yaml_parser_set_scanner_error(yaml_parser_t *parser, const char *context, yaml_mark_t context_mark, const char *problem)
Definition: scanner.c:782
#define IS_BREAKZ(string)
Definition: yaml_private.h:358
#define IS_ALPHA(string)
Definition: yaml_private.h:201
static int yaml_parser_scan_block_scalar_breaks(yaml_parser_t *parser, int *indent, yaml_string_t *breaks, yaml_mark_t start_mark, yaml_mark_t *end_mark)
Definition: scanner.c:2956
yaml_token_delete(yaml_token_t *token)
Free any memory allocated for a token object.
Definition: api.c:578
#define TOKEN_INIT(token, token_type, token_start_mark, token_end_mark)
Definition: yaml_private.h:502
static int yaml_parser_fetch_tag(yaml_parser_t *parser)
Definition: scanner.c:1797
#define YAML_DECLARE(type)
The public API declaration.
Definition: yaml.h:38
A FLOW-MAPPING-START token.
Definition: yaml.h:243
static int yaml_parser_decrease_flow_level(yaml_parser_t *parser)
Definition: scanner.c:1187
#define TAG_TOKEN_INIT(token, token_handle, token_suffix, start_mark, end_mark)
Definition: yaml_private.h:523
#define IS_BLANK_AT(string, offset)
Definition: yaml_private.h:323
yaml_error_type_t error
Error type.
Definition: yaml.h:1089
#define IS_DIGIT(string)
Definition: yaml_private.h:211
static int yaml_parser_scan_tag_handle(yaml_parser_t *parser, int directive, yaml_mark_t start_mark, yaml_char_t **handle)
Definition: scanner.c:2503
#define STREAM_START_TOKEN_INIT(token, token_encoding, start_mark, end_mark)
Definition: yaml_private.h:508
static int yaml_parser_fetch_value(yaml_parser_t *parser)
Definition: scanner.c:1685
struct yaml_parser_s::@80 simple_keys
The stack of simple keys.
#define CHECK_AT(string, octet, offset)
Definition: yaml_private.h:177
struct yaml_parser_s::@76 buffer
The working buffer.
size_t token_number
The number of the token.
Definition: yaml.h:1001
static int yaml_parser_fetch_flow_entry(yaml_parser_t *parser)
Definition: scanner.c:1536
static int yaml_parser_scan_plain_scalar(yaml_parser_t *parser, yaml_token_t *token)
Definition: scanner.c:3386
static int yaml_parser_increase_flow_level(yaml_parser_t *parser)
Definition: scanner.c:1161
A STREAM-END token.
Definition: yaml.h:220
enum yaml_token_type_e yaml_token_type_t
Token types.
yaml_token_t * head
The head of the tokens queue.
Definition: yaml.h:1198
#define ENQUEUE(context, queue, value)
Definition: yaml_private.h:474
An ANCHOR token.
Definition: yaml.h:259
#define JOIN(context, string_a, string_b)
Definition: yaml_private.h:160
#define ANCHOR_TOKEN_INIT(token, token_value, start_mark, end_mark)
Definition: yaml_private.h:519
#define SCALAR_TOKEN_INIT(token, token_value, token_length, token_style, start_mark, end_mark)
Definition: yaml_private.h:528
yaml_mark_t context_mark
The context position.
Definition: yaml.h:1101
static int yaml_parser_fetch_anchor(yaml_parser_t *parser, yaml_token_type_t type)
Definition: scanner.c:1767
A FLOW-MAPPING-END token.
Definition: yaml.h:245
yaml_mark_t problem_mark
The problem position.
Definition: yaml.h:1097
size_t tokens_parsed
The number of tokens fetched from the queue.
Definition: yaml.h:1204
static int yaml_parser_save_simple_key(yaml_parser_t *parser)
Definition: scanner.c:1097
#define STRING_EXTEND(context, string)
Definition: yaml_private.h:148
#define IS_BLANKZ_AT(string, offset)
Definition: yaml_private.h:373
A DOCUMENT-START token.
Definition: yaml.h:227
static int yaml_parser_scan_to_next_token(yaml_parser_t *parser)
Definition: scanner.c:1921
#define STRING_INIT(context, string, size)
Definition: yaml_private.h:135
#define STREAM_END_TOKEN_INIT(token, start_mark, end_mark)
Definition: yaml_private.h:512
static int yaml_parser_scan_tag_uri(yaml_parser_t *parser, int directive, yaml_char_t *head, yaml_mark_t start_mark, yaml_char_t **uri)
Definition: scanner.c:2570
This structure holds information about a potential simple key.
Definition: yaml.h:993
#define MAX_NUMBER_LENGTH
Definition: scanner.c:2198
The plain scalar style.
Definition: yaml.h:169
The literal scalar style.
Definition: yaml.h:177
static int yaml_parser_scan_anchor(yaml_parser_t *parser, yaml_token_t *token, yaml_token_type_t type)
Definition: scanner.c:2322
size_t line
The position line.
Definition: yaml.h:150
#define POP(context, stack)
Definition: yaml_private.h:456
A DOCUMENT-END token.
Definition: yaml.h:229
#define IS_Z(string)
Definition: yaml_private.h:290
static VALUE mark(VALUE self)
Definition: psych_parser.c:512
static int yaml_parser_stale_simple_keys(yaml_parser_t *parser)
Definition: scanner.c:1056
#define CHECK(sub)
Definition: compile.c:408
Cannot scan the input stream.
Definition: yaml.h:132
static int yaml_parser_fetch_key(yaml_parser_t *parser)
Definition: scanner.c:1632
size_t index
The position index.
Definition: yaml.h:147
static int yaml_parser_scan_version_directive_number(yaml_parser_t *parser, yaml_mark_t start_mark, int *number)
Definition: scanner.c:2211
#define IS_BLANKZ(string)
Definition: yaml_private.h:376
yaml_parser_scan(yaml_parser_t *parser, yaml_token_t *token)
Scan the input stream and produce the next token.
Definition: scanner.c:742
const char * name
Definition: nkf.c:208
yaml_malloc(size_t size)
Definition: api.c:31
const char * problem
Error description.
Definition: yaml.h:1091
int possible
Is a simple key possible?
Definition: yaml.h:995
A BLOCK-ENTRY token.
Definition: yaml.h:248
yaml_parser_fetch_more_tokens(yaml_parser_t *parser)
Definition: scanner.c:800
#define memcpy(d, s, n)
Definition: ffi_common.h:55
static int yaml_parser_scan_directive_name(yaml_parser_t *parser, yaml_mark_t start_mark, yaml_char_t **name)
Definition: scanner.c:2113
#define VERSION_DIRECTIVE_TOKEN_INIT(token, token_major, token_minor, start_mark, end_mark)
Definition: yaml_private.h:534
#define IS_BREAK(string)
Definition: yaml_private.h:344
size_t column
The position column.
Definition: yaml.h:153
#define NULL
Definition: _sdbm.c:102
A KEY token.
Definition: yaml.h:252
int flow_level
The number of unclosed &#39;[&#39; and &#39;{&#39; indicators.
Definition: yaml.h:1189
static int yaml_parser_roll_indent(yaml_parser_t *parser, ptrdiff_t column, ptrdiff_t number, yaml_token_type_t type, yaml_mark_t mark)
Definition: scanner.c:1207
#define SKIP_LINE(parser)
Definition: scanner.c:500
The token structure.
Definition: yaml.h:267
#define CLEAR(context, string)
Definition: yaml_private.h:156
static int yaml_parser_fetch_plain_scalar(yaml_parser_t *parser)
Definition: scanner.c:1890
The single-quoted scalar style.
Definition: yaml.h:172