skytools.sql_tokenizer: support fully-qualified names

It's easier to merge name parts here than to do it in upper layers.

The merging does not support some crazier parts of sql syntax
like comments between name parts, so it defaults to off.

diff --git a/python/skytools/parsing.py b/python/skytools/parsing.py
index bdcd1e386edefeb4d7948c5a8e72e1fedc3f916d..0545116bfe354b4ce4c066de160670099f761215 100644 (file)
--- a/python/skytools/parsing.py
+++ b/python/skytools/parsing.py
@@ -220,9 +220,13 @@ def parse_tabbed_table(txt):
 
 _extstr = r""" ['] (?: [^'\\]+ | \\. | [']['] )* ['] """
 _stdstr = r""" ['] (?: [^']+ | [']['] )* ['] """
+_name = r""" (?: [a-z][a-z0-9_$]* | " (?: [^"]+ | "" )* " ) """
+
+_ident   = r""" (?P<ident> %s ) """ % _name
+_fqident = r""" (?P<ident> %s (?: \. %s )? ) """ % (_name, _name)
+
 _base_sql = r"""
-      (?P<ident>  [a-z][a-z0-9_$]* | ["] (?: [^"]+ | ["]["] )* ["] )
-    | (?P<dolq>   (?P<dname> [$] (?: [_a-z][_a-z0-9]*)? [$] )
+      (?P<dolq>   (?P<dname> [$] (?: [_a-z][_a-z0-9]*)? [$] )
                   .*?
                   (?P=dname) )
     | (?P<num>    [0-9][0-9.e]* )
@@ -232,11 +236,18 @@ _base_sql = r"""
     | (?P<ws>     (?: \s+ | [/][*] .*? [*][/] | [-][-][^\n]* )+ )
     | (?P<error>  ['"$\\] )
     | (?P<sym>    . )"""
-_std_sql = r"""(?: (?P<str> [E] %s | %s ) | %s )""" % (_extstr, _stdstr, _base_sql)
-_ext_sql = r"""(?: (?P<str> [E]? %s ) | %s )""" % (_extstr, _base_sql)
+
+_base_sql_fq = r"%s | %s" % (_fqident, _base_sql)
+_base_sql    = r"%s | %s" % (_ident, _base_sql)
+
+_std_sql    = r"""(?: (?P<str> [E] %s | %s ) | %s )""" % (_extstr, _stdstr, _base_sql)
+_std_sql_fq = r"""(?: (?P<str> [E] %s | %s ) | %s )""" % (_extstr, _stdstr, _base_sql_fq)
+_ext_sql    = r"""(?: (?P<str> [E]? %s ) | %s )""" % (_extstr, _base_sql)
+_ext_sql_fq = r"""(?: (?P<str> [E]? %s ) | %s )""" % (_extstr, _base_sql_fq)
 _std_sql_rc = _ext_sql_rc = None
+_std_sql_fq_rc = _ext_sql_fq_rc = None
 
-def sql_tokenizer(sql, standard_quoting = False, ignore_whitespace = False):
+def sql_tokenizer(sql, standard_quoting = False, ignore_whitespace = False, fqident = False):
     r"""Parser SQL to tokens.
 
     Iterator, returns (toktype, tokstr) tuples.
@@ -246,16 +257,26 @@ def sql_tokenizer(sql, standard_quoting = False, ignore_whitespace = False):
     [('ident', 'select'), ('sym', '*'), ('ident', 'from'), ('ident', 'a'), ('sym', '.'), ('ident', 'b')]
     >>> [x for x in sql_tokenizer("\"c olumn\",'str''val'")]
     [('ident', '"c olumn"'), ('sym', ','), ('str', "'str''val'")]
+    >>> list(sql_tokenizer('a.b a."b "" c" a.1', fqident=True, ignore_whitespace=True))
+    [('ident', 'a.b'), ('ident', 'a."b "" c"'), ('ident', 'a'), ('sym', '.'), ('num', '1')]
     """
-    global _std_sql_rc, _ext_sql_rc
+    global _std_sql_rc, _ext_sql_rc, _std_sql_fq_rc, _ext_sql_fq_rc
     if not _std_sql_rc:
         _std_sql_rc = re.compile(_std_sql, re.X | re.I | re.S)
         _ext_sql_rc = re.compile(_ext_sql, re.X | re.I | re.S)
+        _std_sql_fq_rc = re.compile(_std_sql_fq, re.X | re.I | re.S)
+        _ext_sql_fq_rc = re.compile(_ext_sql_fq, re.X | re.I | re.S)
 
     if standard_quoting:
-        rc = _std_sql_rc
+        if fqident:
+            rc = _std_sql_fq_rc
+        else:
+            rc = _std_sql_rc
     else:
-        rc = _ext_sql_rc
+        if fqident:
+            rc = _ext_sql_fq_rc
+        else:
+            rc = _ext_sql_rc
 
     pos = 0
     while 1:
@@ -264,8 +285,9 @@ def sql_tokenizer(sql, standard_quoting = False, ignore_whitespace = False):
             break
         pos = m.end()
         typ = m.lastgroup
-        if not ignore_whitespace or typ != "ws":
-            yield (m.lastgroup, m.group())
+        if ignore_whitespace and typ == "ws":
+            continue
+        yield (typ, m.group())
 
 _copy_from_stdin_re = "copy.*from\s+stdin"
 _copy_from_stdin_rc = None