fix: Less boxed testing (getsentry#522)

untitaker · web-flow · commit 911b89e56348 · 2019-10-07T18:26:25.000+02:00
* fix: No boxed testing

* fix: Un-break django tests

* fix: Django 2.0

* fix: Work around weird pytest (?) bug

* fix: Remove unused imports

* fix: Fix new test

* fix: Fix remainder of Django tests

* fix: Upgrade tox

* wip

* fix: Use more forked markers

* fix: Revert some diffs

* fix: Set db.params

* fix: Remove unused imports

* fix: Fix django tests

* fix: Contextvars are patching

* fix: Fix celery freeze

* fix: Fix threading tests
diff --git a/pytest.ini b/pytest.ini
@@ -1,4 +1,4 @@
 [pytest]
 DJANGO_SETTINGS_MODULE = tests.integrations.django.myapp.settings
-addopts = --boxed --tb=short
+addopts = --tb=short
 markers = tests_internal_exceptions
diff --git a/test-requirements.txt b/test-requirements.txt
@@ -1,6 +1,6 @@
 hypothesis==3.69.9
 pytest==3.7.3
-pytest-xdist==1.23.0
+git+https://github.com/untitaker/pytest-forked@forked-marker#egg=pytest-forked
 tox==3.7.0
 Werkzeug==0.15.3
 pytest-localserver==0.4.1
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -184,15 +184,22 @@ def fast_serialize(request):
 
 
 @pytest.fixture
-def sentry_init(monkeypatch_test_transport, fast_serialize):
+def sentry_init(monkeypatch_test_transport, fast_serialize, request):
     def inner(*a, **kw):
         hub = sentry_sdk.Hub.current
         client = sentry_sdk.Client(*a, **kw)
         client.options["_experiments"]["fast_serialize"] = fast_serialize
         hub.bind_client(client)
         monkeypatch_test_transport(sentry_sdk.Hub.current.client)
 
-    return inner
+    if request.node.get_closest_marker("forked"):
+        # Do not run isolation if the test is already running in
+        # ultimate isolation (seems to be required for celery tests that
+        # fork)
+        yield inner
+    else:
+        with sentry_sdk.Hub(None):
+            yield inner
 
 
 class TestTransport(Transport):
@@ -258,7 +265,11 @@ def read_flush(self):
 
 
 # scope=session ensures that fixture is run earlier
-@pytest.fixture(scope="session", params=[None, "eventlet", "gevent"])
+@pytest.fixture(
+    scope="session",
+    params=[None, "eventlet", "gevent"],
+    ids=("threads", "eventlet", "greenlet"),
+)
 def maybe_monkeypatched_threading(request):
     if request.param == "eventlet":
         try:
diff --git a/tests/integrations/celery/test_celery.py b/tests/integrations/celery/test_celery.py
@@ -271,6 +271,7 @@ def dummy_task(self):
         assert e["type"] == "ZeroDivisionError"
 
 
+@pytest.mark.forked
 @pytest.mark.skipif(VERSION < (4,), reason="in-memory backend broken")
 def test_transport_shutdown(request, celery, capture_events_forksafe, tmpdir):
     events = capture_events_forksafe()
diff --git a/tests/integrations/django/test_basic.py b/tests/integrations/django/test_basic.py
@@ -1,3 +1,5 @@
+from __future__ import absolute_import
+
 import pytest
 import json
 
@@ -81,6 +83,7 @@ def test_transaction_with_class_view(sentry_init, client, capture_events):
     assert event["message"] == "hi"
 
 
+@pytest.mark.forked
 @pytest.mark.django_db
 def test_user_captured(sentry_init, client, capture_events):
     sentry_init(integrations=[DjangoIntegration()], send_default_pii=True)
@@ -102,6 +105,7 @@ def test_user_captured(sentry_init, client, capture_events):
     }
 
 
+@pytest.mark.forked
 @pytest.mark.django_db
 def test_queryset_repr(sentry_init, capture_events):
     sentry_init(integrations=[DjangoIntegration()])
@@ -156,6 +160,7 @@ def test_500(sentry_init, client, capture_events):
     assert content == "Sentry error: %s" % event_id
 
 
+@pytest.mark.forked
 def test_management_command_raises():
     # This just checks for our assumption that Django passes through all
     # exceptions by default, so our excepthook can be used for management
@@ -164,6 +169,7 @@ def test_management_command_raises():
         execute_from_command_line(["manage.py", "mycrash"])
 
 
+@pytest.mark.forked
 @pytest.mark.django_db
 @pytest.mark.parametrize("with_integration", [True, False])
 def test_sql_queries(sentry_init, capture_events, with_integration):
@@ -175,9 +181,16 @@ def test_sql_queries(sentry_init, capture_events, with_integration):
 
     from django.db import connection
 
-    sql = connection.cursor()
+    sentry_init(
+        integrations=[DjangoIntegration()],
+        send_default_pii=True,
+        _experiments={"record_sql_params": True},
+    )
 
     events = capture_events()
+
+    sql = connection.cursor()
+
     with pytest.raises(OperationalError):
         # table doesn't even exist
         sql.execute("""SELECT count(*) FROM people_person WHERE foo = %s""", [123])
@@ -193,6 +206,7 @@ def test_sql_queries(sentry_init, capture_events, with_integration):
         assert crumb["data"]["db.params"] == [123]
 
 
+@pytest.mark.forked
 @pytest.mark.django_db
 def test_sql_dict_query_params(sentry_init, capture_events):
     sentry_init(
@@ -234,6 +248,7 @@ def test_sql_dict_query_params(sentry_init, capture_events):
         lambda sql: sql.SQL('SELECT %(my_param)s FROM "foobar"'),
     ],
 )
+@pytest.mark.forked
 @pytest.mark.django_db
 def test_sql_psycopg2_string_composition(sentry_init, capture_events, query):
     sentry_init(
@@ -262,6 +277,7 @@ def test_sql_psycopg2_string_composition(sentry_init, capture_events, query):
     assert crumb["data"]["db.params"] == {"my_param": 10}
 
 
+@pytest.mark.forked
 @pytest.mark.django_db
 def test_sql_psycopg2_placeholders(sentry_init, capture_events):
     sentry_init(
diff --git a/tests/integrations/threading/test_threading.py b/tests/integrations/threading/test_threading.py
@@ -8,6 +8,7 @@
 from sentry_sdk.integrations.threading import ThreadingIntegration
 
 
+@pytest.mark.forked
 @pytest.mark.parametrize("integrations", [[ThreadingIntegration()], []])
 def test_handles_exceptions(sentry_init, capture_events, integrations):
     sentry_init(default_integrations=False, integrations=integrations)
@@ -30,6 +31,7 @@ def crash():
         assert not events
 
 
+@pytest.mark.forked
 @pytest.mark.parametrize("propagate_hub", (True, False))
 def test_propagates_hub(sentry_init, capture_events, propagate_hub):
     sentry_init(
@@ -85,6 +87,7 @@ def run(self):
     assert not gc.collect()
 
 
+@pytest.mark.forked
 def test_double_patching(sentry_init, capture_events):
     sentry_init(default_integrations=False, integrations=[ThreadingIntegration()])
     events = capture_events()
diff --git a/tests/test_basics.py b/tests/test_basics.py
@@ -205,26 +205,21 @@ def test_breadcrumbs(sentry_init, capture_events):
     assert len(event["breadcrumbs"]) == 0
 
 
-def test_integration_scoping():
+def test_integration_scoping(sentry_init, capture_events):
     logger = logging.getLogger("test_basics")
-    events = []
-    logging_integration = LoggingIntegration(event_level=logging.WARNING)
 
     # This client uses the logging integration
-    client_with_logging = Client(
-        transport=events.append,
-        default_integrations=False,
-        integrations=[logging_integration],
-    )
-    Hub.current.bind_client(client_with_logging)
+    logging_integration = LoggingIntegration(event_level=logging.WARNING)
+    sentry_init(default_integrations=False, integrations=[logging_integration])
+    events = capture_events()
     logger.warning("This is a warning")
+    assert len(events) == 1
 
     # This client does not
-    client_without_logging = Client(transport=events.append, default_integrations=False)
-    Hub.current.bind_client(client_without_logging)
+    sentry_init(default_integrations=False)
+    events = capture_events()
     logger.warning("This is not a warning")
-
-    assert len(events) == 1
+    assert not events
 
 
 def test_client_initialized_within_scope(sentry_init, caplog):
@@ -233,7 +228,7 @@ def test_client_initialized_within_scope(sentry_init, caplog):
     sentry_init(debug=True)
 
     with push_scope():
-        sentry_init()
+        Hub.current.bind_client(Client())
 
     record, = (x for x in caplog.records if x.levelname == "WARNING")
 
diff --git a/tests/test_client.py b/tests/test_client.py
@@ -1,4 +1,5 @@
 # coding: utf-8
+import os
 import json
 import pytest
 import subprocess
@@ -31,6 +32,9 @@ def capture_event(self, event):
 
 
 def test_transport_option(monkeypatch):
+    if "SENTRY_DSN" in os.environ:
+        monkeypatch.delenv("SENTRY_DSN")
+
     dsn = "https://foo@sentry.io/123"
     dsn2 = "https://bar@sentry.io/124"
     assert str(Client(dsn=dsn).dsn) == dsn
diff --git a/tests/test_transport.py b/tests/test_transport.py
@@ -21,6 +21,7 @@ def inner(*args, **kwargs):
     return inner
 
 
+@pytest.mark.forked
 @pytest.mark.parametrize("debug", (True, False))
 @pytest.mark.parametrize("client_flush_method", ["close", "flush"])
 def test_transport_works(
diff --git a/tests/utils/test_contextvars.py b/tests/utils/test_contextvars.py
@@ -1,17 +1,20 @@
+import pytest
 import random
 import time
 
 
 from sentry_sdk.utils import _is_threading_local_monkey_patched
 
 
+@pytest.mark.forked
 def test_thread_local_is_patched(maybe_monkeypatched_threading):
     if maybe_monkeypatched_threading is None:
         assert not _is_threading_local_monkey_patched()
     else:
         assert _is_threading_local_monkey_patched()
 
 
+@pytest.mark.forked
 def test_leaks(maybe_monkeypatched_threading):
     import threading
 
