Storm

Merge lp://qastaging/~jamesh/storm/bug-349482 into lp://qastaging/storm

  1. bug-349482
  2. Merge into trunk
Proposed by James Henstridge
Status: Merged
Merged at revision: not available
Proposed branch: lp://qastaging/~jamesh/storm/bug-349482
Merge into: lp://qastaging/storm
Diff against target: 303 lines
To merge this branch: bzr merge lp://qastaging/~jamesh/storm/bug-349482
Reviewer Review Type Date Requested Status
Gustavo Niemeyer Approve
Thomas Herve (community) Approve
Review via email: mp+6693@code.qastaging.launchpad.net
To post a comment you must log in.
Revision history for this message
James Henstridge (jamesh) wrote :

Change the way compile_python works to pass constants in directly rather than trying to stringify them. Among other things, this makes it possible to handle variables whose values do not have a repr() that is valid Python code.

I also updated the get_column() callback to take a column object rather than going column -> name -> column conversions when getting column values in the matcher.

Revision history for this message
Thomas Herve (therve) wrote :

+ exec code in namespace
+ return namespace['closure'](state.parameters, bool)

Can you add a local dict and do this:

+ local = {}
+ exec code in namespace, local
+ return local['closure'](state.parameters, bool)

This is it will be even cleaner.

+1!

review: Approve
Revision history for this message
Gustavo Niemeyer (niemeyer) wrote :

Nice improvements! Thank you!

James, I recall something about this change being motivated by an actual failing expression. Is it easy to reproduce this case in a test just so that we ensure this case won't ever come back again?

+1!

review: Approve
lp://qastaging/~jamesh/storm/bug-349482 updated
310. By James Henstridge

Add a test to show that get_matcher() works for values whose repr() is
not a valid Python expression.

311. By James Henstridge

Add a NEWS item about changes.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'NEWS'
--- NEWS 2009-06-13 12:18:15 +0000
+++ NEWS 2009-06-18 15:20:10 +0000
@@ -28,6 +28,8 @@
28 - The test suite now passes when run with Python 2.6.28 - The test suite now passes when run with Python 2.6.
29 - ListVariable now converts its elements to database representation29 - ListVariable now converts its elements to database representation
30 correctly (bug #136806 reported by Marc Tardif).30 correctly (bug #136806 reported by Marc Tardif).
31 - compile_python now works for values that don't produce valid Python
32 expressions with repr().
3133
3234
330.14 (2009-01-09)350.14 (2009-01-09)
3436
=== modified file 'storm/expr.py'
--- storm/expr.py 2008-06-27 16:44:28 +0000
+++ storm/expr.py 2009-06-18 15:20:10 +0000
@@ -206,8 +206,18 @@
206class CompilePython(Compile):206class CompilePython(Compile):
207207
208 def get_matcher(self, expr):208 def get_matcher(self, expr):
209 exec "def match(get_column): return bool(%s)" % self(expr)209 state = State()
210 return match210 source = self(expr, state)
211 namespace = {}
212 code = ("def closure(parameters, bool):\n"
213 " [%s] = parameters\n"
214 " def match(get_column):\n"
215 " return bool(%s)\n"
216 " return match" %
217 (",".join("_%d" % i for i in range(len(state.parameters))),
218 source))
219 exec code in namespace
220 return namespace['closure'](state.parameters, bool)
211221
212222
213class State(object):223class State(object):
@@ -348,12 +358,18 @@
348 return "NULL"358 return "NULL"
349359
350360
351@compile_python.when(str, unicode, bool, int, long, float,361@compile_python.when(str, unicode, int, long, float, type(None))
352 datetime, date, time, timedelta, type(None))
353def compile_python_builtin(compile, expr, state):362def compile_python_builtin(compile, expr, state):
354 return repr(expr)363 return repr(expr)
355364
356365
366@compile_python.when(bool, datetime, date, time, timedelta)
367def compile_python_bool_and_dates(compile, expr, state):
368 index = len(state.parameters)
369 state.parameters.append(expr)
370 return "_%d" % index
371
372
357@compile.when(Variable)373@compile.when(Variable)
358def compile_variable(compile, variable, state):374def compile_variable(compile, variable, state):
359 state.parameters.append(variable)375 state.parameters.append(variable)
@@ -361,7 +377,9 @@
361377
362@compile_python.when(Variable)378@compile_python.when(Variable)
363def compile_python_variable(compile, variable, state):379def compile_python_variable(compile, variable, state):
364 return repr(variable.get())380 index = len(state.parameters)
381 state.parameters.append(variable.get())
382 return "_%d" % index
365383
366384
367# --------------------------------------------------------------------385# --------------------------------------------------------------------
@@ -776,7 +794,9 @@
776794
777@compile_python.when(Column)795@compile_python.when(Column)
778def compile_python_column(compile, column, state):796def compile_python_column(compile, column, state):
779 return "get_column(%s)" % repr(column.name)797 index = len(state.parameters)
798 state.parameters.append(column)
799 return "get_column(_%d)" % index
780800
781801
782# --------------------------------------------------------------------802# --------------------------------------------------------------------
783803
=== modified file 'storm/store.py'
--- storm/store.py 2009-02-16 10:44:31 +0000
+++ storm/store.py 2009-06-18 15:20:10 +0000
@@ -1349,11 +1349,8 @@
1349 match = None1349 match = None
1350 else:1350 else:
1351 match = compile_python.get_matcher(self._where)1351 match = compile_python.get_matcher(self._where)
1352 name_to_column = dict(1352 def get_column(column):
1353 (column.name, column)1353 return obj_info.variables[column].get()
1354 for column in self._find_spec.default_cls_info.columns)
1355 def get_column(name, name_to_column=name_to_column):
1356 return obj_info.variables[name_to_column[name]].get()
1357 objects = []1354 objects = []
1358 cls = self._find_spec.default_cls_info.cls1355 cls = self._find_spec.default_cls_info.cls
1359 for obj_info in self._store._iter_alive():1356 for obj_info in self._store._iter_alive():
13601357
=== modified file 'tests/expr.py'
--- tests/expr.py 2008-10-02 15:47:18 +0000
+++ tests/expr.py 2009-06-18 15:20:10 +0000
@@ -1937,8 +1937,10 @@
19371937
1938 def test_compile_sequence(self):1938 def test_compile_sequence(self):
1939 expr = [elem1, Variable(1), (Variable(2), None)]1939 expr = [elem1, Variable(1), (Variable(2), None)]
1940 py_expr = compile_python(expr)1940 state = State()
1941 self.assertEquals(py_expr, "elem1, 1, 2, None")1941 py_expr = compile_python(expr, state)
1942 self.assertEquals(py_expr, "elem1, _0, _1, None")
1943 self.assertEquals(state.parameters, [1, 2])
19421944
1943 def test_compile_invalid(self):1945 def test_compile_invalid(self):
1944 self.assertRaises(CompileError, compile_python, object())1946 self.assertRaises(CompileError, compile_python, object())
@@ -1965,8 +1967,10 @@
1965 self.assertEquals(py_expr, "1L")1967 self.assertEquals(py_expr, "1L")
19661968
1967 def test_bool(self):1969 def test_bool(self):
1968 py_expr = compile_python(True)1970 state = State()
1969 self.assertEquals(py_expr, "True")1971 py_expr = compile_python(True, state)
1972 self.assertEquals(py_expr, "_0")
1973 self.assertEquals(state.parameters, [True])
19701974
1971 def test_float(self):1975 def test_float(self):
1972 py_expr = compile_python(1.1)1976 py_expr = compile_python(1.1)
@@ -1974,23 +1978,31 @@
19741978
1975 def test_datetime(self):1979 def test_datetime(self):
1976 dt = datetime(1977, 5, 4, 12, 34)1980 dt = datetime(1977, 5, 4, 12, 34)
1977 py_expr = compile_python(dt)1981 state = State()
1978 self.assertEquals(py_expr, repr(dt))1982 py_expr = compile_python(dt, state)
1983 self.assertEquals(py_expr, "_0")
1984 self.assertEquals(state.parameters, [dt])
19791985
1980 def test_date(self):1986 def test_date(self):
1981 d = date(1977, 5, 4)1987 d = date(1977, 5, 4)
1982 py_expr = compile_python(d)1988 state = State()
1983 self.assertEquals(py_expr, repr(d))1989 py_expr = compile_python(d, state)
1990 self.assertEquals(py_expr, "_0")
1991 self.assertEquals(state.parameters, [d])
19841992
1985 def test_time(self):1993 def test_time(self):
1986 t = time(12, 34)1994 t = time(12, 34)
1987 py_expr = compile_python(t)1995 state = State()
1988 self.assertEquals(py_expr, repr(t))1996 py_expr = compile_python(t, state)
1997 self.assertEquals(py_expr, "_0")
1998 self.assertEquals(state.parameters, [t])
19891999
1990 def test_timedelta(self):2000 def test_timedelta(self):
1991 td = timedelta(days=1, seconds=2, microseconds=3)2001 td = timedelta(days=1, seconds=2, microseconds=3)
1992 py_expr = compile_python(td)2002 state = State()
1993 self.assertEquals(py_expr, repr(td))2003 py_expr = compile_python(td, state)
2004 self.assertEquals(py_expr, "_0")
2005 self.assertEquals(state.parameters, [td])
19942006
1995 def test_none(self):2007 def test_none(self):
1996 py_expr = compile_python(None)2008 py_expr = compile_python(None)
@@ -1998,63 +2010,87 @@
19982010
1999 def test_column(self):2011 def test_column(self):
2000 expr = Column(column1)2012 expr = Column(column1)
2001 py_expr = compile_python(expr)2013 state = State()
2002 self.assertEquals(py_expr, "get_column('column1')")2014 py_expr = compile_python(expr, state)
2015 self.assertEquals(py_expr, "get_column(_0)")
2016 self.assertEquals(state.parameters, [expr])
20032017
2004 def test_column_table(self):2018 def test_column_table(self):
2005 expr = Column(column1, table1)2019 expr = Column(column1, table1)
2006 py_expr = compile_python(expr)2020 state = State()
2007 self.assertEquals(py_expr, "get_column('column1')")2021 py_expr = compile_python(expr, state)
2022 self.assertEquals(py_expr, "get_column(_0)")
2023 self.assertEquals(state.parameters, [expr])
20082024
2009 def test_variable(self):2025 def test_variable(self):
2010 expr = Variable("value")2026 expr = Variable("value")
2011 py_expr = compile_python(expr)2027 state = State()
2012 self.assertEquals(py_expr, "'value'")2028 py_expr = compile_python(expr, state)
2029 self.assertEquals(py_expr, "_0")
2030 self.assertEquals(state.parameters, ["value"])
20132031
2014 def test_eq(self):2032 def test_eq(self):
2015 expr = Eq(Variable(1), Variable(2))2033 expr = Eq(Variable(1), Variable(2))
2016 py_expr = compile_python(expr)2034 state = State()
2017 self.assertEquals(py_expr, "1 == 2")2035 py_expr = compile_python(expr, state)
2036 self.assertEquals(py_expr, "_0 == _1")
2037 self.assertEquals(state.parameters, [1, 2])
20182038
2019 def test_ne(self):2039 def test_ne(self):
2020 expr = Ne(Variable(1), Variable(2))2040 expr = Ne(Variable(1), Variable(2))
2021 py_expr = compile_python(expr)2041 state = State()
2022 self.assertEquals(py_expr, "1 != 2")2042 py_expr = compile_python(expr, state)
2043 self.assertEquals(py_expr, "_0 != _1")
2044 self.assertEquals(state.parameters, [1, 2])
20232045
2024 def test_gt(self):2046 def test_gt(self):
2025 expr = Gt(Variable(1), Variable(2))2047 expr = Gt(Variable(1), Variable(2))
2026 py_expr = compile_python(expr)2048 state = State()
2027 self.assertEquals(py_expr, "1 > 2")2049 py_expr = compile_python(expr, state)
2050 self.assertEquals(py_expr, "_0 > _1")
2051 self.assertEquals(state.parameters, [1, 2])
20282052
2029 def test_ge(self):2053 def test_ge(self):
2030 expr = Ge(Variable(1), Variable(2))2054 expr = Ge(Variable(1), Variable(2))
2031 py_expr = compile_python(expr)2055 state = State()
2032 self.assertEquals(py_expr, "1 >= 2")2056 py_expr = compile_python(expr, state)
2057 self.assertEquals(py_expr, "_0 >= _1")
2058 self.assertEquals(state.parameters, [1, 2])
20332059
2034 def test_lt(self):2060 def test_lt(self):
2035 expr = Lt(Variable(1), Variable(2))2061 expr = Lt(Variable(1), Variable(2))
2036 py_expr = compile_python(expr)2062 state = State()
2037 self.assertEquals(py_expr, "1 < 2")2063 py_expr = compile_python(expr, state)
2064 self.assertEquals(py_expr, "_0 < _1")
2065 self.assertEquals(state.parameters, [1, 2])
20382066
2039 def test_le(self):2067 def test_le(self):
2040 expr = Le(Variable(1), Variable(2))2068 expr = Le(Variable(1), Variable(2))
2041 py_expr = compile_python(expr)2069 state = State()
2042 self.assertEquals(py_expr, "1 <= 2")2070 py_expr = compile_python(expr, state)
2071 self.assertEquals(py_expr, "_0 <= _1")
2072 self.assertEquals(state.parameters, [1, 2])
20432073
2044 def test_lshift(self):2074 def test_lshift(self):
2045 expr = LShift(Variable(1), Variable(2))2075 expr = LShift(Variable(1), Variable(2))
2046 py_expr = compile_python(expr)2076 state = State()
2047 self.assertEquals(py_expr, "1<<2")2077 py_expr = compile_python(expr, state)
2078 self.assertEquals(py_expr, "_0<<_1")
2079 self.assertEquals(state.parameters, [1, 2])
20482080
2049 def test_rshift(self):2081 def test_rshift(self):
2050 expr = RShift(Variable(1), Variable(2))2082 expr = RShift(Variable(1), Variable(2))
2051 py_expr = compile_python(expr)2083 state = State()
2052 self.assertEquals(py_expr, "1>>2")2084 py_expr = compile_python(expr, state)
2085 self.assertEquals(py_expr, "_0>>_1")
2086 self.assertEquals(state.parameters, [1, 2])
20532087
2054 def test_in(self):2088 def test_in(self):
2055 expr = In(Variable(1), Variable(2))2089 expr = In(Variable(1), Variable(2))
2056 py_expr = compile_python(expr)2090 state = State()
2057 self.assertEquals(py_expr, "1 in (2,)")2091 py_expr = compile_python(expr, state)
2092 self.assertEquals(py_expr, "_0 in (_1,)")
2093 self.assertEquals(state.parameters, [1, 2])
20582094
2059 def test_and(self):2095 def test_and(self):
2060 expr = And(elem1, elem2, And(elem3, elem4))2096 expr = And(elem1, elem2, And(elem3, elem4))
@@ -2109,8 +2145,20 @@
21092145
2110 match = compile_python.get_matcher((col1 > 10) & (col2 < 10))2146 match = compile_python.get_matcher((col1 > 10) & (col2 < 10))
21112147
2112 self.assertTrue(match({"column1": 15, "column2": 5}.get))2148 self.assertTrue(match({col1: 15, col2: 5}.get))
2113 self.assertFalse(match({"column1": 5, "column2": 15}.get))2149 self.assertFalse(match({col1: 5, col2: 15}.get))
2150
2151 def test_match_bad_repr(self):
2152 """The get_matcher() works for expressions containing values
2153 whose repr is not valid Python syntax."""
2154 class BadRepr(object):
2155 def __repr__(self):
2156 return "$Not a valid Python expression$"
2157
2158 value = BadRepr()
2159 col1 = Column(column1)
2160 match = compile_python.get_matcher(col1 == Variable(value))
2161 self.assertTrue(match({col1: value}.get))
21142162
21152163
2116class LazyValueExprTest(TestHelper):2164class LazyValueExprTest(TestHelper):

Subscribers

People subscribed via source and target branches

to status/vote changes: