Restore Str.kind attribute (PR #49)

Guido van Rossum · Guido van Rossum · commit 199ad58db144 · 2019-01-19T17:08:37.000-08:00
diff --git a/ast3/Include/Python-ast.h b/ast3/Include/Python-ast.h
@@ -318,6 +318,7 @@ struct _expr {
 
         struct {
             string s;
+            string kind;
         } Str;
 
         struct {
@@ -601,8 +602,8 @@ expr_ty _Ta3_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int
                   lineno, int col_offset, PyArena *arena);
 #define Num(a0, a1, a2, a3) _Ta3_Num(a0, a1, a2, a3)
 expr_ty _Ta3_Num(object n, int lineno, int col_offset, PyArena *arena);
-#define Str(a0, a1, a2, a3) _Ta3_Str(a0, a1, a2, a3)
-expr_ty _Ta3_Str(string s, int lineno, int col_offset, PyArena *arena);
+#define Str(a0, a1, a2, a3, a4) _Ta3_Str(a0, a1, a2, a3, a4)
+expr_ty _Ta3_Str(string s, string kind, int lineno, int col_offset, PyArena *arena);
 #define FormattedValue(a0, a1, a2, a3, a4, a5) _Ta3_FormattedValue(a0, a1, a2, a3, a4, a5)
 expr_ty _Ta3_FormattedValue(expr_ty value, int conversion, expr_ty format_spec,
                             int lineno, int col_offset, PyArena *arena);
diff --git a/ast3/Parser/Python.asdl b/ast3/Parser/Python.asdl
@@ -77,7 +77,7 @@ module Python
          | Compare(expr left, cmpop* ops, expr* comparators)
          | Call(expr func, expr* args, keyword* keywords)
          | Num(object n) -- a number as a PyObject.
-         | Str(string s) -- need to specify raw, unicode, etc?
+         | Str(string s, string kind) -- need to specify raw, unicode, etc?
          | FormattedValue(expr value, int? conversion, expr? format_spec)
          | JoinedStr(expr* values)
          | Bytes(bytes s)
diff --git a/ast3/Python/Python-ast.c b/ast3/Python/Python-ast.c
@@ -307,8 +307,10 @@ static char *Num_fields[]={
 };
 static PyTypeObject *Str_type;
 _Py_IDENTIFIER(s);
+_Py_IDENTIFIER(kind);
 static char *Str_fields[]={
     "s",
+    "kind",
 };
 static PyTypeObject *FormattedValue_type;
 _Py_IDENTIFIER(conversion);
@@ -988,7 +990,7 @@ static int init_types(void)
     if (!Call_type) return 0;
     Num_type = make_type("Num", expr_type, Num_fields, 1);
     if (!Num_type) return 0;
-    Str_type = make_type("Str", expr_type, Str_fields, 1);
+    Str_type = make_type("Str", expr_type, Str_fields, 2);
     if (!Str_type) return 0;
     FormattedValue_type = make_type("FormattedValue", expr_type,
                                     FormattedValue_fields, 3);
@@ -2186,19 +2188,25 @@ Num(object n, int lineno, int col_offset, PyArena *arena)
 }
 
 expr_ty
-Str(string s, int lineno, int col_offset, PyArena *arena)
+Str(string s, string kind, int lineno, int col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!s) {
         PyErr_SetString(PyExc_ValueError,
                         "field s is required for Str");
         return NULL;
     }
+    if (!kind) {
+        PyErr_SetString(PyExc_ValueError,
+                        "field kind is required for Str");
+        return NULL;
+    }
     p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
     if (!p)
         return NULL;
     p->kind = Str_kind;
     p->v.Str.s = s;
+    p->v.Str.kind = kind;
     p->lineno = lineno;
     p->col_offset = col_offset;
     return p;
@@ -3450,6 +3458,11 @@ ast2obj_expr(void* _o)
         if (_PyObject_SetAttrId(result, &PyId_s, value) == -1)
             goto failed;
         Py_DECREF(value);
+        value = ast2obj_string(o->v.Str.kind);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_kind, value) == -1)
+            goto failed;
+        Py_DECREF(value);
         break;
     case FormattedValue_kind:
         result = PyType_GenericNew(FormattedValue_type, NULL, NULL);
@@ -6998,6 +7011,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
     }
     if (isinstance) {
         string s;
+        string kind;
 
         if (lookup_attr_id(obj, &PyId_s, &tmp) < 0) {
             return 1;
@@ -7012,7 +7026,18 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = Str(s, lineno, col_offset, arena);
+        if (_PyObject_HasAttrId(obj, &PyId_kind)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_kind);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_string(tmp, &kind, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"kind\" missing from Str");
+            return 1;
+        }
+        *out = Str(s, kind, lineno, col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
diff --git a/ast3/Python/ast.c b/ast3/Python/ast.c
@@ -5163,14 +5163,25 @@ FstringParser_Dealloc(FstringParser *state)
 static expr_ty
 make_str_node_and_del(PyObject **str, struct compiling *c, const node* n)
 {
-    PyObject *s = *str;
+    PyObject *kind, *s = *str;
+    const char *raw = STR(CHILD(n, 0));
+    /* currently Python allows up to 2 string modifiers */
+    char *ch, s_kind[3] = {0, 0, 0};
+    ch = s_kind;
+    while (*raw && *raw != '\'' && *raw != '"') {
+        *ch++ = *raw++;
+    }
+    kind = PyUnicode_FromString(s_kind);
+    if (!kind) {
+        return NULL;
+    }
     *str = NULL;
     assert(PyUnicode_CheckExact(s));
     if (PyArena_AddPyObject(c->c_arena, s) < 0) {
         Py_DECREF(s);
         return NULL;
     }
-    return Str(s, LINENO(n), n->n_col_offset, c->c_arena);
+    return Str(s, kind, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
 /* Add a non-f-string (that is, a regular literal string). str is
