• R/O
  • SSH
  • HTTPS

yash: Commit


Commit MetaInfo

Revision3884 (tree)
Time2018-09-19 00:55:26
Authormagicant

Log Message

Refactor successive word parser

The simple command is now parsed by the new parse_simple_command_tokens
function. Successive words are now parsed by the new parse_words
function. The parse_assignments_and_redirects, parse_words_and_redirects
and parse_words_to_paren functions are removed in favor of them.

The is_command_delimiter_tokentype function is removed as it is no
longer used.

Change Summary

Incremental Difference

--- yash/branches/token_based_parser/parser.c (revision 3883)
+++ yash/branches/token_based_parser/parser.c (revision 3884)
@@ -282,8 +282,6 @@
282282 __attribute__((pure));
283283 static tokentype_T identify_reserved_word(const wordunit_T *wu)
284284 __attribute__((pure));
285-static bool is_command_delimiter_tokentype(tokentype_T tt)
286- __attribute__((const));
287285 static bool is_closing_tokentype(tokentype_T tt)
288286 __attribute__((const));
289287
@@ -468,26 +466,6 @@
468466 return identify_reserved_word_string(wu->wu_string);
469467 }
470468
471-/* Determines if the given token delimits a simple command. */
472-bool is_command_delimiter_tokentype(tokentype_T tt)
473-{
474- switch (tt) {
475- case TT_END_OF_INPUT:
476- case TT_NEWLINE:
477- case TT_SEMICOLON:
478- case TT_DOUBLE_SEMICOLON:
479- case TT_AMP:
480- case TT_AMPAMP:
481- case TT_PIPE:
482- case TT_PIPEPIPE:
483- case TT_LPAREN:
484- case TT_RPAREN:
485- return true;
486- default:
487- return false;
488- }
489-}
490-
491469 /* Determines if the specified token is a 'closing' token such as ")", "}", and
492470 * "fi". Closing tokens delimit and/or lists. */
493471 bool is_closing_tokentype(tokentype_T tt)
@@ -581,17 +559,15 @@
581559 __attribute__((nonnull,malloc,warn_unused_result));
582560 static command_T *parse_command(parsestate_T *ps)
583561 __attribute__((nonnull,malloc,warn_unused_result));
584-static redir_T **parse_assignments_and_redirects(parsestate_T *ps, command_T *c)
562+static void **parse_simple_command_tokens(
563+ parsestate_T *ps, assign_T **assigns, redir_T **redirs)
585564 __attribute__((nonnull,malloc,warn_unused_result));
586-static void **parse_words_and_redirects(
587- parsestate_T *ps, redir_T **redirlastp, bool first)
565+static void **parse_words(parsestate_T *ps, bool skip_newlines)
588566 __attribute__((nonnull,malloc,warn_unused_result));
589567 static void parse_redirect_list(parsestate_T *ps, redir_T **lastp)
590568 __attribute__((nonnull));
591569 static assign_T *tryparse_assignment(parsestate_T *ps)
592570 __attribute__((nonnull,malloc,warn_unused_result));
593-static void **parse_words_to_paren(parsestate_T *ps)
594- __attribute__((nonnull,malloc,warn_unused_result));
595571 static redir_T *tryparse_redirect(parsestate_T *ps)
596572 __attribute__((nonnull,malloc,warn_unused_result));
597573 static wordunit_T *parse_word(parsestate_T *ps, bool testfunc(wchar_t c))
@@ -1297,8 +1273,10 @@
12971273 result->refcount = 1;
12981274 result->c_lineno = ps->info->lineno;
12991275 result->c_type = CT_SIMPLE;
1300- redir_T **redirlastp = parse_assignments_and_redirects(ps, result);
1301- result->c_words = parse_words_and_redirects(ps, redirlastp, true);
1276+ result->c_assigns = NULL;
1277+ result->c_redirs = NULL;
1278+ result->c_words = parse_simple_command_tokens(
1279+ ps, &result->c_assigns, &result->c_redirs);
13021280
13031281 if (result->c_words[0] == NULL && result->c_assigns == NULL &&
13041282 result->c_redirs == NULL) {
@@ -1315,67 +1293,72 @@
13151293 return try_reparse_as_function(ps, result);
13161294 }
13171295
1318-/* Parses assignments and redirections.
1319- * Tokens but the first one are subject to any-type alias substitution,
1320- * including the word just after the parsed assignments and redirections.
1321- * The results are assigned to `c->c_assigns' and `c->c_redirs'.
1322- * The return value is a pointer to the `next' member of the last resultant
1323- * redirection (redir_T). If no redirections were parsed, the result value is a
1324- * pointer to `c->c_redirs'. */
1325-redir_T **parse_assignments_and_redirects(parsestate_T *ps, command_T *c)
1296+/* Parses assignments, redirections, and words.
1297+ * Assignments are parsed before words are parsed. Tokens are subject to
1298+ * any-type alias substitution until the first word is parsed, except for the
1299+ * first token which is parsed intact. The other words are subject to global
1300+ * alias substitution. Redirections can appear anywhere.
1301+ * Parsed Assignments and redirections are assigned to `*assigns' and `redirs',
1302+ * respectively. They must have been initialized NULL (or anything) before
1303+ * calling this function. Parsed words are returned as a newly-malloced
1304+ * NULL-terminated array of pointers to newly-malloced wordunit_T's. */
1305+void **parse_simple_command_tokens(
1306+ parsestate_T *ps, assign_T **assigns, redir_T **redirs)
13261307 {
1327- assign_T **assgnlastp = &c->c_assigns;
1328- redir_T **redirlastp = &c->c_redirs;
1329- assign_T *assgn;
1330- redir_T *redir;
1308+ bool is_first = true;
1309+ plist_T words;
1310+ pl_init(&words);
13311311
1332- c->c_assigns = NULL;
1333- c->c_redirs = NULL;
1334- while (!is_command_delimiter_tokentype(ps->tokentype)) {
1335- if ((redir = tryparse_redirect(ps)) != NULL) {
1336- *redirlastp = redir;
1337- redirlastp = &redir->next;
1338- } else if ((assgn = tryparse_assignment(ps)) != NULL) {
1339- *assgnlastp = assgn;
1340- assgnlastp = &assgn->next;
1341- } else {
1342- break;
1312+next:
1313+ if (is_first)
1314+ is_first = false;
1315+ else
1316+ psubstitute_alias_recursive(ps, words.length == 0 ? AF_NONGLOBAL : 0);
1317+
1318+ redir_T *redir = tryparse_redirect(ps);
1319+ if (redir != NULL) {
1320+ *redirs = redir;
1321+ redirs = &redir->next;
1322+ goto next;
1323+ }
1324+
1325+ if (words.length == 0) {
1326+ assign_T *assign = tryparse_assignment(ps);
1327+ if (assign != NULL) {
1328+ *assigns = assign;
1329+ assigns = &assign->next;
1330+ goto next;
13431331 }
1344- psubstitute_alias_recursive(ps, AF_NONGLOBAL);
13451332 }
1346- return redirlastp;
1333+
1334+ if (ps->token != NULL) {
1335+ pl_add(&words, ps->token), ps->token = NULL;
1336+ next_token(ps);
1337+ goto next;
1338+ }
1339+
1340+ return pl_toary(&words);
13471341 }
13481342
1349-/* Parses words and redirections.
1350- * The parsing result of redirections is assigned to `*redirlastp'
1351- * The parsing result of assignments is returned as an array of pointers to
1352- * word units that are cast to (void *).
1353- * `*redirlastp' must have been initialized to NULL beforehand.
1354- * All words are subject to global alias substitution. If `first' is true,
1355- * however, alias substitution is not performed on the first word. */
1356-void **parse_words_and_redirects(
1357- parsestate_T *ps, redir_T **redirlastp, bool first)
1343+/* Parses words.
1344+ * The resultant words are returned as a newly-malloced NULL-terminated array of
1345+ * pointers to word units that are cast to (void *).
1346+ * All words are subject to global alias substitution.
1347+ * If `skip_newlines' is true, newline operators are skipped.
1348+ * Words are parsed until an operator token is found. */
1349+void **parse_words(parsestate_T *ps, bool skip_newlines)
13581350 {
13591351 plist_T wordlist;
1360- redir_T *redir;
1361- wordunit_T *word;
13621352
1363- assert(*redirlastp == NULL);
13641353 pl_init(&wordlist);
1365- while (!is_command_delimiter_tokentype(ps->tokentype)) {
1366- if (!first)
1367- psubstitute_alias_recursive(ps, 0);
1368- if ((redir = tryparse_redirect(ps)) != NULL) {
1369- *redirlastp = redir;
1370- redirlastp = &redir->next;
1371- } else if ((word = ps->token) != NULL) {
1372- ps->token = NULL;
1373- pl_add(&wordlist, word);
1374- next_token(ps);
1375- first = false;
1376- } else {
1354+ for (;;) {
1355+ psubstitute_alias_recursive(ps, 0);
1356+ if (skip_newlines && parse_newline_list(ps))
1357+ continue;
1358+ if (ps->token == NULL)
13771359 break;
1378- }
1360+ pl_add(&wordlist, ps->token), ps->token = NULL;
1361+ next_token(ps);
13791362 }
13801363 return pl_toary(&wordlist);
13811364 }
@@ -1439,7 +1422,7 @@
14391422 /* array assignment */
14401423 next_token(ps);
14411424 result->a_type = A_ARRAY;
1442- result->a_array = parse_words_to_paren(ps);
1425+ result->a_array = parse_words(ps, true);
14431426 if (ps->tokentype == TT_RPAREN)
14441427 next_token(ps);
14451428 else
@@ -1448,28 +1431,6 @@
14481431 return result;
14491432 }
14501433
1451-/* Parses words until the next closing parentheses.
1452- * Delimiter characters other than ')' and '\n' are not allowed.
1453- * Returns a newly malloced array of pointers to newly malloced `wordunit_T's.*/
1454-void **parse_words_to_paren(parsestate_T *ps)
1455-{
1456- plist_T list;
1457-
1458- pl_init(&list);
1459- while (psubstitute_alias_recursive(ps, 0), ps->tokentype != TT_RPAREN) {
1460- if (ps->tokentype == TT_NEWLINE) {
1461- next_line(ps);
1462- next_token(ps);
1463- continue;
1464- }
1465- if (ps->token == NULL)
1466- break;
1467- pl_add(&list, ps->token), ps->token = NULL;
1468- next_token(ps);
1469- }
1470- return pl_toary(&list);
1471-}
1472-
14731434 /* If there is a redirection at the current position, parses and returns it.
14741435 * Otherwise, returns NULL without moving the position. */
14751436 redir_T *tryparse_redirect(parsestate_T *ps)
@@ -2323,13 +2284,8 @@
23232284 parse_in:;
23242285 bool on_next_line = parse_newline_list(ps);
23252286 if (ps->tokentype == TT_IN) {
2326- redir_T *redirs = NULL;
23272287 next_token(ps);
2328- result->c_forwords = parse_words_and_redirects(ps, &redirs, false);
2329- if (redirs != NULL) {
2330- serror(ps, Ngt("redirections are not allowed after `in'"));
2331- redirsfree(redirs);
2332- }
2288+ result->c_forwords = parse_words(ps, false);
23332289 if (ps->tokentype == TT_SEMICOLON)
23342290 next_token(ps);
23352291 } else if (psubstitute_alias(ps, 0)) {
Show on old repository browser