From 655116d41a863c902c1f7957e4c29ba990dd2583 Mon Sep 17 00:00:00 2001 From: Marko Kreen Date: Thu, 25 Nov 2010 14:04:43 +0200 Subject: [PATCH] skytools.sql_tokenizer: optionally return token position --- python/skytools/parsing.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/skytools/parsing.py b/python/skytools/parsing.py index 0545116b..d45bb839 100644 --- a/python/skytools/parsing.py +++ b/python/skytools/parsing.py @@ -247,7 +247,8 @@ _ext_sql_fq = r"""(?: (?P [E]? %s ) | %s )""" % (_extstr, _base_sql_fq) _std_sql_rc = _ext_sql_rc = None _std_sql_fq_rc = _ext_sql_fq_rc = None -def sql_tokenizer(sql, standard_quoting = False, ignore_whitespace = False, fqident = False): +def sql_tokenizer(sql, standard_quoting = False, ignore_whitespace = False, + fqident = False, show_location = False): r"""Parser SQL to tokens. Iterator, returns (toktype, tokstr) tuples. @@ -287,7 +288,10 @@ def sql_tokenizer(sql, standard_quoting = False, ignore_whitespace = False, fqid typ = m.lastgroup if ignore_whitespace and typ == "ws": continue - yield (typ, m.group()) + if show_location: + yield (typ, m.group(), pos) + else: + yield (typ, m.group()) _copy_from_stdin_re = "copy.*from\s+stdin" _copy_from_stdin_rc = None -- 2.39.5